LMS-1612 delete already registered data sets in case of an exception

SVN: 16980

eu_basysbio/etc/service.properties

@@ -12,6 +12,8 @@ commandqueue-dir =
 # Port
 port = 8890
 
+use-ssl = false
+
 # Session timeout in minutes
 session-timeout = 720
 
@@ -98,7 +100,7 @@ quiet-period = 10
 maintenance-plugins = data-set-clean-up
 
 data-set-clean-up.class = ch.systemsx.cisd.etlserver.plugins.DataSetDeletionMaintenanceTask
-data-set-clean-up.interval = 30
+data-set-clean-up.interval = 300
 data-set-clean-up.data-source = data-source
 
 post-registration-upload.class = eu.basysbio.cisd.dss.PostRegistrationDatabaseUploadTask

eu_basysbio/source/java/eu/basysbio/cisd/dss/TimeSeriesAndTimePointDataSetHandler.java

@@ -27,6 +27,7 @@ import org.apache.log4j.Logger;
 
 import ch.rinn.restrictions.Private;
 import ch.systemsx.cisd.common.Constants;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.mail.IMailClient;
@@ -116,12 +117,13 @@ public class TimeSeriesAndTimePointDataSetHandler implements IDataSetHandler
             count++;
         }
 
-        void logAndSendOptinallyAnEMail(Logger logger, IMailClient mailClient, boolean sendEMail)
+        void logSendEMailAndHandlerError(Logger logger, IMailClient mailClient, boolean sendEMail)
         {
             if (count < numberOfDerivedDataSets)
             {
-                operationLog.error("Only " + count + " " + type.name + " data sets instead of "
-                        + numberOfDerivedDataSets + " have been registered.");
+                int numberOfFailures = numberOfDerivedDataSets - count;
+                operationLog.error(numberOfFailures + " " + type.name
+                        + " data sets couldn't be registered.");
                 if (sendEMail)
                 {
                     String subject =
@@ -130,14 +132,16 @@ public class TimeSeriesAndTimePointDataSetHandler implements IDataSetHandler
                             Constants.DATE_FORMAT.get().format(
                                     new Date(timeProvider.getTimeInMilliseconds()));
                     String message =
-                            "Uploading of data set '" + dataSetFileName + "' failed because only "
-                                    + count + " of " + numberOfDerivedDataSets + " " + type.name
-                                    + " data sets could be registered in openBIS.\n\n"
+                            "Uploading of data set '" + dataSetFileName + "' failed because "
+                                    + numberOfFailures + " of " + numberOfDerivedDataSets + " "
+                                    + type.name + " data sets couldn't be registered.\n\n"
                                     + "Please, contact the help desk for support: "
                                     + HELPDESK_EMAIL + "\n(Time stamp of failure: " + timeStamp
                                     + ")";
                     mailClient.sendMessage(subject, message, null, null, userEMail, HELPDESK_EMAIL);
                 }
+                throw new UserFailureException("Not all data sets could be registered. "
+                        + "For more details see error messages in the log.");
             } else
             {
                 if (count > 0 && operationLog.isInfoEnabled())
@@ -181,10 +185,13 @@ public class TimeSeriesAndTimePointDataSetHandler implements IDataSetHandler
 
     private final ITimeProvider timeProvider;
 
+    private final IEncapsulatedOpenBISService service;
+
     public TimeSeriesAndTimePointDataSetHandler(Properties parentProperties,
             IDataSetHandler delegator, IEncapsulatedOpenBISService openbisService)
     {
         this.delegator = delegator;
+        this.service = openbisService;
         this.mailClient = new MailClient(parentProperties);
         Properties specificProperties =
                 ExtendedProperties.getSubset(parentProperties,
@@ -198,11 +205,13 @@ public class TimeSeriesAndTimePointDataSetHandler implements IDataSetHandler
     }
 
     @Private
-    TimeSeriesAndTimePointDataSetHandler(IDataSetHandler delegator, IMailClient mailClient,
+    TimeSeriesAndTimePointDataSetHandler(IDataSetHandler delegator,
+            IEncapsulatedOpenBISService openbisService, IMailClient mailClient,
             IDataSetHandler timePointDataSetHandler, File timePointDataSetFolder,
             ITimeProvider timeProvider)
     {
         this.delegator = delegator;
+        service = openbisService;
         this.mailClient = mailClient;
         this.timePointDataSetHandler = timePointDataSetHandler;
         this.timePointDataSetFolder = timePointDataSetFolder;
@@ -217,20 +226,36 @@ public class TimeSeriesAndTimePointDataSetHandler implements IDataSetHandler
         boolean successful = result.isEmpty() == false;
         if (successful)
         {
-            DataSetInformation dataSetInformation = result.get(0);
-            MessageBuilder builder =
+            try
+            {
+                DataSetInformation dataSetInformation = result.get(0);
+                MessageBuilder builder =
                     new MessageBuilder(dataSetInformation.tryGetUploadingUserEmail(), timeProvider);
-            builder.setDataSetFileName(dataSet);
-            File[] files = timePointDataSetFolder.listFiles();
-            handleDerivedDataSets(files, TypeOfDerivedDataSet.TIME_POINT, timePointDataSetHandler,
-                    dataSetInfos, builder);
-            files = dataSet.getParentFile().listFiles(LCA_MIC_TIME_SERIES_FILE_FILTER);
-            handleDerivedDataSets(files, TypeOfDerivedDataSet.LCA_MIC_TIME_SERIES, delegator,
-                    dataSetInfos, builder);
-            DataSetType dataSetType = dataSetInformation.getDataSetType();
-            boolean lcaMicTimeSeries =
+                builder.setDataSetFileName(dataSet);
+                File[] files = timePointDataSetFolder.listFiles();
+                handleDerivedDataSets(files, TypeOfDerivedDataSet.TIME_POINT, timePointDataSetHandler,
+                        dataSetInfos, builder);
+                files = dataSet.getParentFile().listFiles(LCA_MIC_TIME_SERIES_FILE_FILTER);
+                handleDerivedDataSets(files, TypeOfDerivedDataSet.LCA_MIC_TIME_SERIES, delegator,
+                        dataSetInfos, builder);
+                DataSetType dataSetType = dataSetInformation.getDataSetType();
+                boolean lcaMicTimeSeries =
                     dataSetType.getCode().equals(DataSetHandler.LCA_MIC_TIME_SERIES);
-            builder.logAndSendOptinallyAnEMail(operationLog, mailClient, lcaMicTimeSeries == false);
+                builder.logSendEMailAndHandlerError(operationLog, mailClient, lcaMicTimeSeries == false);
+            } catch (RuntimeException ex)
+            {
+                // Data sets are deleted in reverse order because of possible parent-child relationships.
+                for (int i = dataSetInfos.size() - 1; i >= 0; i--)
+                {
+                    String dataSetCode = dataSetInfos.get(i).getDataSetCode();
+                    if (operationLog.isInfoEnabled())
+                    {
+                        operationLog.info("Delete data set " + dataSetCode);
+                    }
+                    service.deleteDataSet(dataSetCode, "Rollback registration");
+                }
+                throw ex;
+            }
         }
         return dataSetInfos;
     }

eu_basysbio/sourceTest/java/eu/basysbio/cisd/dss/TimeSeriesAndTimePointDataSetHandlerTest.java

@@ -29,17 +29,20 @@ import org.hamcrest.collection.IsArray;
 import org.hamcrest.core.IsNull;
 import org.jmock.Expectations;
 import org.jmock.Mockery;
+import org.jmock.Sequence;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
 import org.testng.annotations.Test;
 
 import ch.rinn.restrictions.Friend;
 import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
 import ch.systemsx.cisd.common.logging.LogInitializer;
 import ch.systemsx.cisd.common.mail.From;
 import ch.systemsx.cisd.common.mail.IMailClient;
 import ch.systemsx.cisd.common.utilities.ITimeProvider;
 import ch.systemsx.cisd.etlserver.IDataSetHandler;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
 import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
 
@@ -85,6 +88,8 @@ public class TimeSeriesAndTimePointDataSetHandlerTest extends AbstractFileSystem
     private File dropBox;
     private File timePointFolder;
     private ITimeProvider timeProvider;
+    private IEncapsulatedOpenBISService service;
+    private Sequence sequence;
 
     @BeforeMethod
     public void beforeMethod() throws Exception
@@ -92,7 +97,9 @@ public class TimeSeriesAndTimePointDataSetHandlerTest extends AbstractFileSystem
         super.setUp();
         LogInitializer.init();
         context = new Mockery();
+        sequence = context.sequence("sequence");
         delegator = context.mock(IDataSetHandler.class, "delegator");
+        service = context.mock(IEncapsulatedOpenBISService.class);
         timePointDataSetHandler = context.mock(IDataSetHandler.class, "timePointDataSetHandler");
         mailClient = context.mock(IMailClient.class);
         timeProvider = context.mock(ITimeProvider.class);
@@ -100,7 +107,7 @@ public class TimeSeriesAndTimePointDataSetHandlerTest extends AbstractFileSystem
         timePointFolder = new File(workingDirectory, "time-point-folder");
         timePointFolder.mkdirs();
         handler =
-                new TimeSeriesAndTimePointDataSetHandler(delegator, mailClient,
+                new TimeSeriesAndTimePointDataSetHandler(delegator, service, mailClient,
                         timePointDataSetHandler, timePointFolder, timeProvider);
     }
 
@@ -177,13 +184,15 @@ public class TimeSeriesAndTimePointDataSetHandlerTest extends AbstractFileSystem
         final File tp2 = new File(timePointFolder, "tp2");
         tp2.createNewFile();
         final DataSetInformation ds2 = new DataSetInformation();
+        ds2.setDataSetCode("ds2");
         final DataSetInformation dataSetInformation = new DataSetInformation();
+        dataSetInformation.setDataSetCode("ds1");
         dataSetInformation.setUploadingUserEmail("john.doe@abc.de");
         dataSetInformation.setDataSetType(new DataSetType(DataSetHandler.TIME_SERIES));
         String expectedSubject = "BaSysBio: Failed uploading of data set 'drop-box'";
         String expectedMessage =
                 "Uploading of data set 'drop-box' failed "
-                        + "because only 1 of 2 time point data sets could be registered in openBIS.\n\n"
+                        + "because 1 of 2 time point data sets couldn't be registered.\n\n"
                         + "Please, contact the help desk for support: " + HELPDESK_EMAIL + "\n"
                         + "(Time stamp of failure: 1970-01-01 01:01:14 +0100)";
         prepareSendingEMail(expectedSubject, expectedMessage, dataSetInformation, true);
@@ -201,14 +210,24 @@ public class TimeSeriesAndTimePointDataSetHandlerTest extends AbstractFileSystem
 
                     one(timeProvider).getTimeInMilliseconds();
                     will(returnValue(42 * 42 * 42L));
+                    
+                    one(service).deleteDataSet("ds2", "Rollback registration");
+                    inSequence(sequence);
+                    one(service).deleteDataSet("ds1", "Rollback registration");
+                    inSequence(sequence);
                 }
             });
 
-        List<DataSetInformation> dataSets = handler.handleDataSet(dropBox);
+        try
+        {
+            handler.handleDataSet(dropBox);
+            fail("UserFailureException expected");
+        } catch (UserFailureException ex)
+        {
+            assertEquals("Not all data sets could be registered. "
+                    + "For more details see error messages in the log.", ex.getMessage());
+        }
 
-        assertEquals(2, dataSets.size());
-        assertSame(dataSetInformation, dataSets.get(0));
-        assertSame(ds2, dataSets.get(1));
         context.assertIsSatisfied();
     }
 
@@ -220,13 +239,15 @@ public class TimeSeriesAndTimePointDataSetHandlerTest extends AbstractFileSystem
         final File tp2 = new File(workingDirectory, DataSetHandler.LCA_MIC_TIME_SERIES + "2");
         tp2.createNewFile();
         final DataSetInformation ds2 = new DataSetInformation();
+        ds2.setDataSetCode("ds2");
         final DataSetInformation dataSetInformation = new DataSetInformation();
+        dataSetInformation.setDataSetCode("ds1");
         dataSetInformation.setUploadingUserEmail("john.doe@abc.de");
         dataSetInformation.setDataSetType(new DataSetType(DataSetHandler.LCA_MIC));
         String expectedSubject = "BaSysBio: Failed uploading of data set 'drop-box'";
         String expectedMessage =
             "Uploading of data set 'drop-box' failed "
-            + "because only 1 of 2 LCA MIC time series data sets could be registered in openBIS.\n\n"
+            + "because 1 of 2 LCA MIC time series data sets couldn't be registered.\n\n"
             + "Please, contact the help desk for support: " + HELPDESK_EMAIL + "\n"
                         + "(Time stamp of failure: 1970-01-01 01:01:14 +0100)";
         prepareSendingEMail(expectedSubject, expectedMessage, dataSetInformation, true);
@@ -244,14 +265,23 @@ public class TimeSeriesAndTimePointDataSetHandlerTest extends AbstractFileSystem
 
                     one(timeProvider).getTimeInMilliseconds();
                     will(returnValue(42 * 42 * 42L));
+                    
+                    one(service).deleteDataSet("ds2", "Rollback registration");
+                    inSequence(sequence);
+                    one(service).deleteDataSet("ds1", "Rollback registration");
+                    inSequence(sequence);
                 }
             });
 
-        List<DataSetInformation> dataSets = handler.handleDataSet(dropBox);
-        
-        assertEquals(2, dataSets.size());
-        assertSame(dataSetInformation, dataSets.get(0));
-        assertSame(ds2, dataSets.get(1));
+        try
+        {
+            handler.handleDataSet(dropBox);
+            fail("UserFailureException expected");
+        } catch (UserFailureException ex)
+        {
+            assertEquals("Not all data sets could be registered. "
+                    + "For more details see error messages in the log.", ex.getMessage());
+        }
         context.assertIsSatisfied();
     }