[LMS-2205] Rsync Archiver - allow to choose between simple and paranoid checks

SVN: 21027

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/RemoteDataSetFileOperationsExecutor.java

@@ -192,8 +192,7 @@ public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOp
         {
             return BooleanStatus.createFalse("Data set location '" + dataSet + "' doesn't exist");
         }
-        BooleanStatus existsStatus =
- executor.exists(destination.getPath(), timeoutInMillis);
+        BooleanStatus existsStatus = executor.exists(destination.getPath(), timeoutInMillis);
         if (false == existsStatus.isSuccess())
         {
             return existsStatus;
@@ -203,8 +202,7 @@ public final class RemoteDataSetFileOperationsExecutor implements IDataSetFileOp
         Map<String, Long> dataSetFileSizesByPaths =
                 FolderFileSizesReportGenerator.extractSizesByPaths(storeFiles, dataSet);
         String cmd = createListFilesWithFileSizeCmd(destination.getPath(), gfindExecutable);
-        ProcessResult result =
- executor.executeCommandRemotely(cmd, timeoutInMillis);
+        ProcessResult result = executor.executeCommandRemotely(cmd, timeoutInMillis);
 
         if (result.isOK() == false)
         {

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java

@@ -63,13 +63,20 @@ public class DemoArchiver extends AbstractArchiverProcessingPlugin
     }
 
     @Override
-    public BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset,
+    public BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset,
             ArchiverTaskContext context)
     {
         boolean present = archiveContents.contains(dataset.getDataSetCode());
         return BooleanStatus.createFromBoolean(present);
     }
 
+    @Override
+    protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset)
+    {
+        boolean present = archiveContents.contains(dataset.getDataSetCode());
+        return BooleanStatus.createFromBoolean(present);
+    }
+
     @Override
     public DatasetProcessingStatuses doDeleteFromArchive(List<DatasetLocation> dataSets)
     {
@@ -83,5 +90,4 @@ public class DemoArchiver extends AbstractArchiverProcessingPlugin
         }
         return statuses;
     }
-
 }

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java

@@ -74,17 +74,23 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
 
     private static final long serialVersionUID = 1L;
 
-    @Private public static final String SHARE_FINDER_KEY = "share-finder";
-    
+    @Private
+    public static final String SHARE_FINDER_KEY = "share-finder";
+
+    private static final String SYNCHRONIZE_ARCHIVE = "synchronize-archive";
+
     private final IStatusChecker archivePrerequisiteOrNull;
 
     private final IStatusChecker unarchivePrerequisiteOrNull;
-    
+
+    private final boolean synchronizeArchive;
+
     private transient IShareIdManager shareIdManager;
-    
+
     private transient IEncapsulatedOpenBISService service;
-    
-    @Private transient IDataSetStatusUpdater statusUpdater;
+
+    @Private
+    transient IDataSetStatusUpdater statusUpdater;
 
     public AbstractArchiverProcessingPlugin(Properties properties, File storeRoot,
             IStatusChecker archivePrerequisiteOrNull, IStatusChecker unarchivePrerequisiteOrNull)
@@ -92,8 +98,13 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
         super(properties, storeRoot);
         this.archivePrerequisiteOrNull = archivePrerequisiteOrNull;
         this.unarchivePrerequisiteOrNull = unarchivePrerequisiteOrNull;
+
+        properties.getProperty(SYNCHRONIZE_ARCHIVE, Boolean.TRUE.toString());
+        this.synchronizeArchive =
+                Boolean.parseBoolean(properties.getProperty(SYNCHRONIZE_ARCHIVE,
+                        Boolean.TRUE.toString()));
     }
-    
+
     /**
      * NOTE: this method is not allowed to throw exception as this will leave data sets in the
      * openBIS database with an inconsistent status.
@@ -103,11 +114,9 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
 
     /**
      * NOTE: this method is not allowed to throw exception as this will leave data sets in the
-     * openBIS database with an inconsistent status.
-     * 
-     * Implementations of this method should invoke
-     * <code>context.getUnarchivingPreparation().prepareForUnarchiving()</code> for each
-     * data set before doing the actual unarchiving.
+     * openBIS database with an inconsistent status. Implementations of this method should invoke
+     * <code>context.getUnarchivingPreparation().prepareForUnarchiving()</code> for each data set
+     * before doing the actual unarchiving.
      */
     abstract protected DatasetProcessingStatuses doUnarchive(List<DatasetDescription> datasets,
             ArchiverTaskContext context);
@@ -117,13 +126,19 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
      * exist in the openBIS database.
      */
     abstract protected DatasetProcessingStatuses doDeleteFromArchive(List<DatasetLocation> datasets);
-    
+
+    /**
+     * @return <code>true</code> if the dataset is present and synchronized with the archive,
+     *         <code>false</code> otherwise.
+     */
+    abstract protected BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset,
+            ArchiverTaskContext context);
+
     /**
      * @return <code>true</code> if the dataset is present in the archive, <code>false</code>
      *         otherwise.
      */
-    abstract protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset,
-            ArchiverTaskContext context);
+    abstract protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset);
 
     public ProcessingStatus archive(List<DatasetDescription> datasets,
             final ArchiverTaskContext context, boolean removeFromDataStore)
@@ -194,16 +209,18 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
             final ArchiverTaskContext context, boolean removeFromDataStore)
     {
 
-        GroupedDatasets groupedDatasets = groupByPresenceInArchive(datasets, context);
+        GroupedDatasets groupedDatasets =
+                groupByPresenceInArchive(datasets, context, synchronizeArchive);
+
         List<DatasetDescription> notPresentInArchive = groupedDatasets.getNotPresentAsList();
         DatasetProcessingStatuses statuses = new DatasetProcessingStatuses();
-        if (notPresentInArchive.isEmpty() == false)
+        if (!notPresentInArchive.isEmpty())
         {
             // copy data sets in the archive
             statuses = doArchive(notPresentInArchive, context);
 
             // paranoid check to make sure everything really got archived
-            groupedDatasets = groupByPresenceInArchive(datasets, context);
+            groupedDatasets = groupByPresenceInArchive(datasets, context, true);
         }
 
         if (removeFromDataStore)
@@ -211,7 +228,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
             // only remove the when we are sure we have got a backup in the archive
             removeFromDataStore(groupedDatasets.getPresentInArchive(), context);
         }
-        
+
         // merge the archiver statuses with the paranoid check results
         return mergeArchiveStatuses(statuses, groupedDatasets);
     }
@@ -263,7 +280,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
         IDataSetDeleter dataSetDeleter = ServiceProvider.getDataStoreService().getDataSetDeleter();
         dataSetDeleter.scheduleDeletionOfDataSets(datasets);
     }
-    
+
     public ProcessingStatus unarchive(List<DatasetDescription> datasets, ArchiverTaskContext context)
     {
         operationLog.info("Unarchiving of the following datasets has been requested: "
@@ -499,26 +516,30 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
         }
     }
 
-    protected GroupedDatasets groupByPresenceInArchive(List<DatasetDescription> datasets,
-            ArchiverTaskContext context)
+    private GroupedDatasets groupByPresenceInArchive(List<DatasetDescription> datasets,
+            ArchiverTaskContext context, boolean checkIfSynchronized)
     {
         List<DatasetDescription> present = new ArrayList<DatasetDescription>();
-        Map<DatasetDescription, BooleanStatus> notPresent = new HashMap<DatasetDescription, BooleanStatus>();
-        
-        for (DatasetDescription dataset : datasets) 
+        Map<DatasetDescription, BooleanStatus> notPresent =
+                new HashMap<DatasetDescription, BooleanStatus>();
+
+        for (DatasetDescription dataset : datasets)
         {
-            BooleanStatus presentStatus = isDataSetPresentInArchive(dataset, context);
+            BooleanStatus presentStatus =
+                    checkIfSynchronized ? isDataSetSynchronizedWithArchive(dataset, context)
+                            : isDataSetPresentInArchive(dataset);
             if (presentStatus.isSuccess())
             {
                 present.add(dataset);
-            } else {
+            } else
+            {
                 notPresent.put(dataset, presentStatus);
             }
         }
-        
+
         return new GroupedDatasets(present, notPresent);
     }
-    
+
     private IShareIdManager getShareIdManager()
     {
         if (shareIdManager == null)
@@ -527,7 +548,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
         }
         return shareIdManager;
     }
-    
+
     private IShareFinder getShareFinder()
     {
         Properties props =
@@ -545,8 +566,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
 
         return shareFinder;
     }
-    
-    
+
     private IEncapsulatedOpenBISService getService()
     {
         if (service == null)

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java

@@ -117,7 +117,7 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
             File sshExecutable = Copier.getExecutable(properties, SSH_EXEC);
             File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC);
             File gfindExecutable = Copier.getExecutable(properties, GFIND_EXEC);
-            
+
             IPathCopier copier =
                     pathCopierFactory.create(rsyncExecutable, sshExecutable, timeoutInMillis);
             copier.check();
@@ -212,10 +212,10 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
     }
 
     /**
-     * Checks if specified dataset's data are present in the destination specified in constructor.
-     * The path at the destination is defined by original location of the data set.
+     * Checks if specified dataset's data are present and synchronized in the destination specified
+     * in constructor. The path at the destination is defined by original location of the data set.
      */
-    public BooleanStatus isPresentInDestination(File originalData, DatasetDescription dataset)
+    public BooleanStatus isSynchronizedWithDestination(File originalData, DatasetDescription dataset)
     {
         try
         {
@@ -231,7 +231,28 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
         {
             return BooleanStatus.createError(ex.getStatus().tryGetErrorMessage());
         }
+    }
 
+    /**
+     * Checks if specified dataset's data are present in the destination specified in constructor.
+     * The path at the destination is defined by original location of the data set.
+     */
+    public BooleanStatus isPresentInDestination(DatasetDescription dataset)
+    {
+        try
+        {
+            File destinationFolder = new File(destination, dataset.getDataSetLocation());
+            BooleanStatus resultStatus = executor.exists(destinationFolder);
+            String message = resultStatus.tryGetMessage();
+            if (message != null) // if there is a message something went wrong
+            {
+                operationLog.error(message);
+            }
+            return resultStatus;
+        } catch (ExceptionWithStatus ex)
+        {
+            return BooleanStatus.createError(ex.getStatus().tryGetErrorMessage());
+        }
     }
 
     private void checkDestinationExists(File destinationFolder)
@@ -266,5 +287,4 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
         }
         return destinationExists;
     }
-
 }
\ No newline at end of file

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java

@@ -24,8 +24,6 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DatasetLocation;
 import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
 
 /**
- * 
- *
  * @author Franz-Josef Elmer
  */
 public interface IDataSetFileOperationsManager
@@ -50,10 +48,15 @@ public interface IDataSetFileOperationsManager
     public abstract Status deleteFromDestination(DatasetLocation dataset);
 
     /**
-     * Checks if specified dataset's data are present in the destination specified in constructor.
-     * The path at the destination is defined by original location of the data set.
+     * Checks if specified dataset's data are present and synchronized in the destination specified
+     * in constructor. The path at the destination is defined by original location of the data set.
      */
-    public abstract BooleanStatus isPresentInDestination(File originalData,
+    public abstract BooleanStatus isSynchronizedWithDestination(File originalData,
             DatasetDescription dataset);
 
+    /**
+     * Checks if specified dataset's data are present in the destination specified in constructor.
+     * The path at the destination is defined by original location of the data set.
+     */
+    public abstract BooleanStatus isPresentInDestination(DatasetDescription dataset);
 }
\ No newline at end of file

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java

@@ -105,11 +105,17 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
     }
 
     @Override
-    protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset,
+    protected BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset,
             ArchiverTaskContext context)
     {
         File originalData = getDatasetDirectory(context, dataset);
-        return fileOperationsManager.isPresentInDestination(originalData, dataset);
+        return fileOperationsManager.isSynchronizedWithDestination(originalData, dataset);
+    }
+
+    @Override
+    protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset)
+    {
+        return fileOperationsManager.isPresentInDestination(dataset);
     }
 
     private Status doArchive(DatasetDescription dataset, File originalData)
@@ -131,4 +137,5 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
     {
         return context.getDirectoryProvider().getDataSetDirectory(dataset);
     }
+
 }

rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java

@@ -213,13 +213,20 @@ public class MLArchiverTask extends AbstractArchiverProcessingPlugin
     }
 
     @Override
-    public BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset,
+    public BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset,
             ArchiverTaskContext context)
     {
         // data is always present, since there is no archive
         return BooleanStatus.createTrue();
     }
 
+    @Override
+    protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset)
+    {
+        // data is always present, since there is no archive
+        return BooleanStatus.createTrue();
+    }
+    
     @Override
     protected void removeFromDataStore(List<DatasetDescription> datasets,
             ArchiverTaskContext context)