SP-45, BIS-21 fill recovery marker file with more info

-  number of retries
-  last retry date
-  the path to the recovery state

SVN: 25304

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java

@@ -481,10 +481,12 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
                         .extractPrecommittedCheckpoint(recoveryMarkerFile);
 
         // then we should ensure that the recovery will actually take place itself!
-        final File recoveryFile =
+        final DataSetStorageRecoveryInfo recoveryInfo =
                 state.getGlobalState().getStorageRecoveryManager()
                         .getRecoveryFileFromMarker(recoveryMarkerFile);
 
+        final File recoveryFile = recoveryInfo.getRecoveryStateFile();
+        
         if (false == recoveryFile.exists())
         {
             // TODO: is it safe to throw from here?
@@ -519,13 +521,13 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
 
         PostRegistrationCleanUpAction cleanupAction =
                 new PostRegistrationCleanUpAction(recoveryState.getIncomingDataSetFile(),
-                        recoveryMarkerFileCleanupAction);
+                        new DoNothingDelegatedAction());
 
-        handleRecoveryState(recoveryState, cleanupAction);
+        handleRecoveryState(recoveryState, cleanupAction,  recoveryMarkerFileCleanupAction);
     }
 
     private void handleRecoveryState(DataSetStoragePrecommitRecoveryState<T> recoveryState,
-            final IDelegatedActionWithResult<Boolean> cleanAfterwardsAction)
+            final IDelegatedActionWithResult<Boolean> cleanAfterwardsAction, final IDelegatedActionWithResult<Boolean> recoveryMarkerCleanup)
     {
         // TODO: Jobs left to do here:
         // rollback
@@ -538,6 +540,10 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
         // rollback delegate
         final List<Throwable> encounteredErrors = new ArrayList<Throwable>();
 
+        //keeps track of whether we should keep or delete the recovery files.
+        //we can delete if succesfully recovered, or rolledback, or gave up
+        boolean shouldDeleteRecoveryFiles = false;
+        
         IRollbackDelegate<T> rollbackDelegate = new IRollbackDelegate<T>()
             {
                 public void didRollbackStorageAlgorithmRunner(
@@ -605,6 +611,8 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
                                 ErrorType.OPENBIS_REGISTRATION_FAILURE);
                 operationLog.info(rollbacker.getErrorMessageForLog());
                 rollbacker.doRollback(logger);
+                
+                shouldDeleteRecoveryFiles = true;
                 //
                 // invokeRollbackTransactionFunction
                 //
@@ -616,6 +624,8 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
                 runner.storeAfterRegistration();
                 logger.registerSuccess();
                 registrationSuccessful = true;
+                
+                shouldDeleteRecoveryFiles = true;
             }
         } catch (org.jmock.api.ExpectationError r)
         {
@@ -632,8 +642,9 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat
 
         logger.logDssRegistrationResult(encounteredErrors);
 
-        // TODO: recreate this clean afterwards function
         cleanAfterwardsAction.execute(registrationSuccessful);
+        
+        recoveryMarkerCleanup.execute(shouldDeleteRecoveryFiles);        
     }
 
     /**

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryInfo.java

+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.registrator;
+
+import java.io.File;
+import java.io.Serializable;
+import java.util.Date;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+
+/**
+ * Simple data type with information about recovery state.
+ *  - the file
+ *  - time of last recovery try
+ *  - total count of tries to recover
+ * 
+ * @author jakubs
+ */
+public class DataSetStorageRecoveryInfo implements Serializable
+{
+    private static final long serialVersionUID = 1L;
+
+    private final File recoveryStateFile;
+    
+    private final Date lastTry;
+    
+    private final int tryCount;
+
+    public DataSetStorageRecoveryInfo(File recoveryStateFile, Date lastTry, int tryCount)
+    {
+        super();
+        this.recoveryStateFile = recoveryStateFile;
+        this.lastTry = lastTry;
+        this.tryCount = tryCount;
+    }
+
+    public File getRecoveryStateFile()
+    {
+        return recoveryStateFile;
+    }
+
+    public Date getLastTry()
+    {
+        return lastTry;
+    }
+
+    public int getTryCount()
+    {
+        return tryCount;
+    }
+    
+    public void writeToFile(File informationFile)
+    {
+        FileUtilities.writeToFile(informationFile, this);
+    }
+    
+    public static DataSetStorageRecoveryInfo loadFromFile(File informationFile)
+    {
+        return FileUtilities.loadToObject(informationFile, DataSetStorageRecoveryInfo.class);
+    }
+}

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageRecoveryManager.java

@@ -17,6 +17,7 @@
 package ch.systemsx.cisd.etlserver.registrator;
 
 import java.io.File;
+import java.util.Date;
 
 import org.apache.log4j.Logger;
 
@@ -60,7 +61,10 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
         FileUtilities.writeToFile(serializedFile, recoveryState);
 
         File processingMarkerFile = getProcessingMarkerFile(runner);
-        FileUtilities.writeToFile(processingMarkerFile, serializedFile.getAbsolutePath());
+        
+        DataSetStorageRecoveryInfo info = new DataSetStorageRecoveryInfo(serializedFile, new Date(), 0);
+        
+        info.writeToFile(processingMarkerFile);
 
         operationLog.info("Store precommit recovery checkpoint with markerfile "
                 + processingMarkerFile);
@@ -95,19 +99,17 @@ public class DataSetStorageRecoveryManager implements IDataSetStorageRecoveryMan
         return new File(dropboxRecoveryStateDir, incomingFileName + PRECOMMIT_SERIALIZED);
     }
 
-    public File getRecoveryFileFromMarker(File markerFile)
+    public DataSetStorageRecoveryInfo getRecoveryFileFromMarker(File markerFile)
     {
-        // trim is necessary as it reads the \n at the end of the file
-        String recoveryFilePath = FileUtilities.loadToString(markerFile).trim();
-        return new File(recoveryFilePath);
+        return DataSetStorageRecoveryInfo.loadFromFile(markerFile);
     }
 
     @SuppressWarnings("unchecked")
     public <T extends DataSetInformation> DataSetStoragePrecommitRecoveryState<T> extractPrecommittedCheckpoint(
             File markerFile)
     {
-        File recoveryFile = getRecoveryFileFromMarker(markerFile);
-        return FileUtilities.loadToObject(recoveryFile, DataSetStoragePrecommitRecoveryState.class);
+        DataSetStorageRecoveryInfo info = getRecoveryFileFromMarker(markerFile);
+        return FileUtilities.loadToObject(info.getRecoveryStateFile(), DataSetStoragePrecommitRecoveryState.class);
     }
 
     public <T extends DataSetInformation> void registrationCompleted(

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetStorageRecoveryManager.java

@@ -57,7 +57,7 @@ public interface IDataSetStorageRecoveryManager
     /**
      * Extracts the recovery file from the marker file
      */
-    File getRecoveryFileFromMarker(File markerFile);
+    DataSetStorageRecoveryInfo getRecoveryFileFromMarker(File markerFile);
     
     /**
      * @return the path of the recovery marker file for the given incoming

datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonDropboxRecoveryTest.java

@@ -181,6 +181,7 @@ public class JythonDropboxRecoveryTest extends AbstractJythonDataSetHandlerTest
         } else
         {
             assertDataSetNotStoredProcess(DATA_SET_CODE);
+            
             assertNoOriginalMarkerFileExists();
             assertNoRecoveryMarkerFile();
             // assert there is no recovery file
@@ -211,7 +212,7 @@ public class JythonDropboxRecoveryTest extends AbstractJythonDataSetHandlerTest
 
     private void assertNoOriginalMarkerFileExists()
     {
-        assertFalse("The original registration marker file should be deleted", markerFile.exists());
+        assertFalse("The original registration marker " +markerFile+ " file should be deleted", markerFile.exists());
     }
 
     private File assertRecoveryMarkerFile()