diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/AbstractPostRegistrationTaskForPhysicalDataSets.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/AbstractPostRegistrationTaskForPhysicalDataSets.java
new file mode 100644
index 0000000000000000000000000000000000000000..af7c460188ffee407db58945751edd08314fd05b
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/AbstractPostRegistrationTaskForPhysicalDataSets.java
@@ -0,0 +1,58 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.postregistration;
+
+import java.util.Properties;
+
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+
+/**
+ * Abstract super class for post registration of physical data sets.
+ *
+ * @author Franz-Josef Elmer
+ */
+public abstract class AbstractPostRegistrationTaskForPhysicalDataSets extends AbstractPostRegistrationTask
+{
+
+ public AbstractPostRegistrationTaskForPhysicalDataSets(Properties properties,
+ IEncapsulatedOpenBISService service)
+ {
+ super(properties, service);
+ }
+
+ /**
+ * do not allow concurrent maintenance tasks to run if they alter the data store contents.
+ */
+ @Override
+ public boolean requiresDataStoreLock()
+ {
+ return true;
+ }
+
+ @Override
+ public final IPostRegistrationTaskExecutor createExecutor(String dataSetCode, boolean container)
+ {
+ if (container)
+ {
+ return DummyPostRegistrationTaskExecutor.INSTANCE;
+ }
+ return createExecutor(dataSetCode);
+ }
+
+ protected abstract IPostRegistrationTaskExecutor createExecutor(String dataSetCode);
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingExecutor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingExecutor.java
new file mode 100644
index 0000000000000000000000000000000000000000..ec0ff72cc8dc2ca5703c87aba4e9b9fa2eef92cd
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingExecutor.java
@@ -0,0 +1,247 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.postregistration;
+
+import static ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus.AVAILABLE;
+import static ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus.BACKUP_PENDING;
+
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.logging.LogLevel;
+import ch.systemsx.cisd.etlserver.plugins.AutoArchiverTask;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IArchiverPlugin;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ProcessingStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DatasetLocation;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+import ch.systemsx.cisd.openbis.generic.shared.translator.DataSetTranslator;
+
+/**
+ * A post-registration task executor that archives data sets.
+ *
+ * @author Kaloyan Enimanev
+ */
+class ArchivingExecutor implements IPostRegistrationTaskExecutor
+{
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+ ArchivingExecutor.class);
+
+ private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
+ ArchivingExecutor.class);
+
+ private final String dataSetCode;
+
+ private final IEncapsulatedOpenBISService service;
+
+ private final IDataSetDirectoryProvider dataSetDirectoryProvider;
+
+ private final IHierarchicalContentProvider hierarchicalContentProvider;
+
+ private final IArchiverPlugin archiver;
+
+ private final boolean updateStatus;
+
+ ArchivingExecutor(String dataSetCode, boolean updateStatus, IEncapsulatedOpenBISService service,
+ IArchiverPlugin archiver, IDataSetDirectoryProvider dataSetDirectoryProvider,
+ IHierarchicalContentProvider hierarchicalContentProvider)
+ {
+ this.dataSetCode = dataSetCode;
+ this.updateStatus = updateStatus;
+ this.service = service;
+ this.archiver = archiver;
+ this.dataSetDirectoryProvider = dataSetDirectoryProvider;
+ this.hierarchicalContentProvider = hierarchicalContentProvider;
+
+ }
+
+ /**
+ * archives the dataset for the specified dataset code.
+ */
+ @Override
+ public void execute()
+ {
+
+ if (archiver == null)
+ {
+ // no archiver is configured
+ operationLog
+ .error("Post-registration archiving cannot be completed, because there is "
+ + "no archiver configured. Please configure an archiver and restart. ");
+ return;
+ }
+
+ AbstractExternalData dataSet = tryGetExternalData(dataSetCode, service);
+ if (dataSet == null)
+ {
+ operationLog.warn("Data set '" + dataSetCode
+ + "' is no longer available in openBIS."
+ + "Archiving post-registration task will be skipped...");
+ return;
+ }
+
+ boolean statusUpdated = updateStatus ?
+ service.compareAndSetDataSetStatus(dataSetCode, AVAILABLE, BACKUP_PENDING,
+ false) : true;
+ if (statusUpdated)
+ {
+ DatasetDescription dataSetDescription =
+ DataSetTranslator.translateToDescription(dataSet);
+ List<DatasetDescription> dataSetAsList =
+ Collections.singletonList(dataSetDescription);
+ ArchiverTaskContext context =
+ new ArchiverTaskContext(dataSetDirectoryProvider,
+ hierarchicalContentProvider);
+ ProcessingStatus processingStatus = archiver.archive(dataSetAsList, context, false);
+ if (false == processingStatus.getErrorStatuses().isEmpty())
+ {
+ notifyAdministrator(processingStatus);
+ }
+ service.compareAndSetDataSetStatus(dataSetCode, BACKUP_PENDING, AVAILABLE, true);
+ }
+ }
+
+ private void notifyAdministrator(ProcessingStatus processingStatus)
+ {
+ StringBuilder message = new StringBuilder();
+ String failedMessage =
+ String.format("Eager archiving of dataset '%s' has failed. \n", dataSetCode);
+ message.append(failedMessage);
+ for (Status status : processingStatus.getErrorStatuses())
+ {
+ if (status.tryGetErrorMessage() != null)
+ {
+ message.append("Error encountered : " + status.tryGetErrorMessage());
+ message.append("\n");
+ }
+ }
+ String footer =
+ String.format("If you wish to archive the dataset in the future, "
+ + "you can configure an '%s'.", AutoArchiverTask.class.getSimpleName());
+ message.append(footer);
+
+ notificationLog.error(message);
+ }
+
+ @Override
+ public ICleanupTask createCleanupTask()
+ {
+ return new ArchivingCleanupTask(dataSetCode, updateStatus, service, archiver);
+ }
+
+ private static class ArchivingCleanupTask implements ICleanupTask
+ {
+ private static final long serialVersionUID = 1L;
+
+ private final String dataSetCode;
+
+ private transient IEncapsulatedOpenBISService service;
+
+ private transient IArchiverPlugin archiver;
+
+ private final boolean updateStatus;
+
+ ArchivingCleanupTask(String dataSetCode, boolean updateStatus, IEncapsulatedOpenBISService service,
+ IArchiverPlugin archiver)
+ {
+ this.dataSetCode = dataSetCode;
+ this.updateStatus = updateStatus;
+ this.service = service;
+ this.archiver = archiver;
+ }
+
+ @Override
+ public void cleanup(ISimpleLogger logger)
+ {
+ if (updateStatus)
+ {
+ boolean statusUpdated =
+ getService().compareAndSetDataSetStatus(dataSetCode, BACKUP_PENDING, AVAILABLE,
+ false);
+
+ if (statusUpdated == false)
+ {
+ // invalid data set status, do not continue
+ return;
+ }
+ }
+
+ DatasetDescription dataSet = tryGetDatasetWithLocation(dataSetCode, getService());
+ if (getArchiver() != null && dataSet != null && dataSet.getDataSetLocation() != null)
+ {
+ DatasetLocation dataset = new DatasetLocation();
+ dataset.setDatasetCode(dataSetCode);
+ dataset.setDataSetLocation(dataSet.getDataSetLocation());
+
+ List<DatasetLocation> dataSetAsList = Collections.singletonList(dataset);
+ getArchiver().deleteFromArchive(dataSetAsList);
+ logger.log(LogLevel.INFO, "Successfully cleaned up leftovers from incomplete "
+ + "archiving of dataset '" + dataSetCode + "'.");
+ }
+ }
+
+ private IEncapsulatedOpenBISService getService()
+ {
+ if (service == null)
+ {
+ service = ServiceProvider.getOpenBISService();
+ }
+ return service;
+ }
+
+ private IArchiverPlugin getArchiver()
+ {
+ if (archiver == null)
+ {
+ archiver = ServiceProvider.getDataStoreService().getArchiverPlugin();
+ }
+ return archiver;
+ }
+ }
+
+ private static AbstractExternalData tryGetExternalData(String dataSetCode,
+ IEncapsulatedOpenBISService service)
+ {
+ List<String> codeAsList = Collections.singletonList(dataSetCode);
+ List<AbstractExternalData> dataList = service.listDataSetsByCode(codeAsList);
+ if (dataList == null || dataList.isEmpty())
+ {
+ return null;
+ }
+
+ AbstractExternalData data = dataList.get(0);
+ return data;
+ }
+
+ private static DatasetDescription tryGetDatasetWithLocation(String dataSetCode,
+ IEncapsulatedOpenBISService service)
+ {
+ AbstractExternalData data = tryGetExternalData(dataSetCode, service);
+ return (data != null) ? DataSetTranslator.translateToDescription(data) : null;
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingPostRegistrationTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingPostRegistrationTask.java
index 8c11106bee6501e35b7afe2d4a38dfebe341f9e8..e48f26b4f5f1fd99d2146ccc26686436abde6f52 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingPostRegistrationTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/ArchivingPostRegistrationTask.java
@@ -16,215 +16,39 @@
package ch.systemsx.cisd.etlserver.postregistration;
-import static ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus.AVAILABLE;
-import static ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus.BACKUP_PENDING;
-import java.util.Collections;
-import java.util.List;
import java.util.Properties;
-import org.apache.log4j.Logger;
-
-import ch.systemsx.cisd.common.exceptions.Status;
-import ch.systemsx.cisd.common.logging.ISimpleLogger;
-import ch.systemsx.cisd.common.logging.LogCategory;
-import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.common.logging.LogLevel;
-import ch.systemsx.cisd.etlserver.plugins.AutoArchiverTask;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
import ch.systemsx.cisd.openbis.dss.generic.shared.IArchiverPlugin;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataStoreServiceInternal;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ProcessingStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DatasetLocation;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
-import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
-import ch.systemsx.cisd.openbis.generic.shared.translator.DataSetTranslator;
/**
- * A post-registration task that archives datasets.
+ * A post-registration task that archives data sets.
*
* @author Kaloyan Enimanev
*/
-public class ArchivingPostRegistrationTask extends AbstractPostRegistrationTask
+public class ArchivingPostRegistrationTask extends AbstractPostRegistrationTaskForPhysicalDataSets
{
- private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
- ArchivingPostRegistrationTask.class);
-
- private static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
- ArchivingPostRegistrationTask.class);
-
public ArchivingPostRegistrationTask(Properties properties, IEncapsulatedOpenBISService service)
{
super(properties, service);
}
- /**
- * do not allow concurrent maintenance tasks to run if they alter the data store contents.
- */
@Override
- public boolean requiresDataStoreLock()
- {
- return true;
- }
-
- @Override
- public IPostRegistrationTaskExecutor createExecutor(String dataSetCode, boolean container)
- {
- if (container)
- {
- return DummyPostRegistrationTaskExecutor.INSTANCE;
- }
- return new Executor(dataSetCode);
- }
-
- private final class Executor implements IPostRegistrationTaskExecutor
+ public IPostRegistrationTaskExecutor createExecutor(String dataSetCode)
{
- private final String dataSetCode;
-
- Executor(String dataSetCode)
- {
- this.dataSetCode = dataSetCode;
-
- }
-
- /**
- * archives the dataset for the specified dataset code.
- */
- @Override
- public void execute()
- {
-
- IArchiverPlugin archiver = ServiceProvider.getDataStoreService().getArchiverPlugin();
- if (archiver == null)
- {
- // no archiver is configured
- operationLog
- .error("Post-registration archiving cannot be completed, because there is "
- + "no archiver configured. Please configure an archiver and restart. ");
- return;
- }
-
- AbstractExternalData dataSet = tryGetExternalData(dataSetCode, service);
- if (dataSet == null)
- {
- operationLog.warn("Data set '" + dataSetCode
- + "' is no longer available in openBIS."
- + "Archiving post-registration task will be skipped...");
- return;
- }
-
- boolean statusUpdated =
- service.compareAndSetDataSetStatus(dataSetCode, AVAILABLE, BACKUP_PENDING,
- false);
- if (statusUpdated)
- {
- DatasetDescription dataSetDescription =
- DataSetTranslator.translateToDescription(dataSet);
- List<DatasetDescription> dataSetAsList =
- Collections.singletonList(dataSetDescription);
- ProcessingStatus processingStatus =
- archiver.archive(dataSetAsList, createArchiverContext(), false);
- if (false == processingStatus.getErrorStatuses().isEmpty())
- {
- notifyAdministrator(processingStatus);
- }
- service.compareAndSetDataSetStatus(dataSetCode, BACKUP_PENDING, AVAILABLE, true);
- }
- }
-
- private void notifyAdministrator(ProcessingStatus processingStatus)
- {
- StringBuilder message = new StringBuilder();
- String failedMessage =
- String.format("Eager archiving of dataset '%s' has failed. \n", dataSetCode);
- message.append(failedMessage);
- for (Status status : processingStatus.getErrorStatuses())
- {
- if (status.tryGetErrorMessage() != null)
- {
- message.append("Error encountered : " + status.tryGetErrorMessage());
- message.append("\n");
- }
- }
- String footer =
- String.format("If you wish to archive the dataset in the future, "
- + "you can configure an '%s'.", AutoArchiverTask.class.getSimpleName());
- message.append(footer);
-
- notificationLog.error(message);
- }
-
- @Override
- public ICleanupTask createCleanupTask()
- {
- return new ArchivingCleanupTask(dataSetCode);
- }
+ IDataStoreServiceInternal dataStoreService = ServiceProvider.getDataStoreService();
+ IArchiverPlugin archiver = dataStoreService.getArchiverPlugin();
+ IDataSetDirectoryProvider dataSetDirectoryProvider =
+ dataStoreService.getDataSetDirectoryProvider();
+ IHierarchicalContentProvider hierarchicalContentProvider =
+ ServiceProvider.getHierarchicalContentProvider();
+ return new ArchivingExecutor(dataSetCode, true, service, archiver, dataSetDirectoryProvider,
+ hierarchicalContentProvider);
}
- private static AbstractExternalData tryGetExternalData(String dataSetCode,
- IEncapsulatedOpenBISService service)
- {
- List<String> codeAsList = Collections.singletonList(dataSetCode);
- List<AbstractExternalData> dataList = service.listDataSetsByCode(codeAsList);
- if (dataList == null || dataList.isEmpty())
- {
- return null;
- }
-
- AbstractExternalData data = dataList.get(0);
- return data;
- }
-
- private static DatasetDescription tryGetDatasetWithLocation(String dataSetCode,
- IEncapsulatedOpenBISService service)
- {
- AbstractExternalData data = tryGetExternalData(dataSetCode, service);
- return (data != null) ? DataSetTranslator.translateToDescription(data) : null;
- }
-
- private static ArchiverTaskContext createArchiverContext()
- {
- return new ArchiverTaskContext(ServiceProvider.getDataStoreService()
- .getDataSetDirectoryProvider(), ServiceProvider.getHierarchicalContentProvider());
- }
-
- private static class ArchivingCleanupTask implements ICleanupTask
- {
- private static final long serialVersionUID = 1L;
-
- private final String dataSetCode;
-
- ArchivingCleanupTask(String dataSetCode)
- {
- this.dataSetCode = dataSetCode;
- }
-
- @Override
- public void cleanup(ISimpleLogger logger)
- {
- IEncapsulatedOpenBISService openBISService = ServiceProvider.getOpenBISService();
- boolean statusUpdated = openBISService
- .compareAndSetDataSetStatus(dataSetCode, BACKUP_PENDING, AVAILABLE, false);
-
- if (statusUpdated == false) {
- // invalid data set status, do not continue
- return;
- }
-
- IArchiverPlugin archiver = ServiceProvider.getDataStoreService().getArchiverPlugin();
- DatasetDescription dataSet = tryGetDatasetWithLocation(dataSetCode, openBISService);
- if (archiver != null && dataSet != null && dataSet.getDataSetLocation() != null)
- {
- DatasetLocation dataset = new DatasetLocation();
- dataset.setDatasetCode(dataSetCode);
- dataset.setDataSetLocation(dataSet.getDataSetLocation());
-
- List<DatasetLocation> dataSetAsList = Collections.singletonList(dataset);
- archiver.deleteFromArchive(dataSetAsList);
- logger.log(LogLevel.INFO, "Successfully cleaned up leftovers from incomplete "
- + "archiving of dataset '" + dataSetCode + "'.");
- }
- }
- }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
index 374c2eeb8ae990632eb08adc7ceae7317dd1e885..a6ce79cee568f27cb95c033443ab94f5a61784ff 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
@@ -57,7 +57,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
*
* @author Franz-Josef Elmer
*/
-public class EagerShufflingTask extends AbstractPostRegistrationTask
+public class EagerShufflingTask extends AbstractPostRegistrationTaskForPhysicalDataSets
{
@Private
public static final String SHARE_FINDER_KEY = "share-finder";
@@ -165,12 +165,6 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
verifyChecksum = PropertyUtils.getBoolean(properties, VERIFY_CHECKSUM_KEY, true);
}
- @Override
- public boolean requiresDataStoreLock()
- {
- return true;
- }
-
private IChecksumProvider getChecksumProvider()
{
if (verifyChecksum)
@@ -183,12 +177,8 @@ public class EagerShufflingTask extends AbstractPostRegistrationTask
}
@Override
- public IPostRegistrationTaskExecutor createExecutor(String dataSetCode, boolean container)
+ public IPostRegistrationTaskExecutor createExecutor(String dataSetCode)
{
- if (container)
- {
- return DummyPostRegistrationTaskExecutor.INSTANCE;
- }
return new Executor(dataSetCode);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/Hdf5CompressingPostRegistrationTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/Hdf5CompressingPostRegistrationTask.java
index f979778db417c2b5f431acb4325b8da1fd13cb4d..4308ea02a0b3e7cdd56d99a3961d1032443f838d 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/Hdf5CompressingPostRegistrationTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/Hdf5CompressingPostRegistrationTask.java
@@ -76,7 +76,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFa
*
* @author Kaloyan Enimanev
*/
-public class Hdf5CompressingPostRegistrationTask extends AbstractPostRegistrationTask
+public class Hdf5CompressingPostRegistrationTask extends AbstractPostRegistrationTaskForPhysicalDataSets
{
public static final String DATA_SET_TYPES = "data-set-types";
@@ -97,22 +97,9 @@ public class Hdf5CompressingPostRegistrationTask extends AbstractPostRegistratio
new HashSet<String>(PropertyUtils.tryGetList(properties, DATA_SET_TYPES));
}
- /**
- * do not allow concurrent maintenance tasks to run if they alter the data store contents.
- */
- @Override
- public boolean requiresDataStoreLock()
- {
- return true;
- }
-
@Override
- public IPostRegistrationTaskExecutor createExecutor(String dataSetCode, boolean container)
+ public IPostRegistrationTaskExecutor createExecutor(String dataSetCode)
{
- if (container)
- {
- return DummyPostRegistrationTaskExecutor.INSTANCE;
- }
return new Executor(dataSetCode);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/SecondCopyPostRegistrationTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/SecondCopyPostRegistrationTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..2db42b3db1b4737f87a9df102014d92f9aa1925a
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/SecondCopyPostRegistrationTask.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.postregistration;
+
+import java.io.File;
+import java.util.List;
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.AbstractArchiverProcessingPlugin;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.DataSetFileOperationsManager;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.IDataSetFileOperationsManager;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.RsyncArchiveCopierFactory;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.RsyncArchiver;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.SshCommandExecutorFactory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IArchiverPlugin;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetStatusUpdater;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataStoreServiceInternal;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
+
+/**
+ * Creates a second copy of the data set.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class SecondCopyPostRegistrationTask extends AbstractPostRegistrationTaskForPhysicalDataSets
+{
+
+ private final DataSetFileOperationsManager fileOperationManager;
+ private final IHierarchicalContentProvider hierarchicalContentProvider;
+ private final IDataSetDirectoryProvider dataSetDirectoryProvider;
+ private final IArchiverPlugin archiver;
+
+ public SecondCopyPostRegistrationTask(Properties properties, IEncapsulatedOpenBISService service)
+ {
+ this(properties, service, ServiceProvider.getDataStoreService(), ServiceProvider
+ .getHierarchicalContentProvider());
+ }
+
+ SecondCopyPostRegistrationTask(Properties properties, IEncapsulatedOpenBISService service,
+ IDataStoreServiceInternal dataStoreService,
+ IHierarchicalContentProvider hierarchicalContentProvider)
+ {
+ super(properties, service);
+ this.hierarchicalContentProvider = hierarchicalContentProvider;
+ fileOperationManager = new DataSetFileOperationsManager(properties,
+ new RsyncArchiveCopierFactory(), new SshCommandExecutorFactory());
+ if (fileOperationManager.isHosted())
+ {
+ throw new ConfigurationFailureException(
+ "Destination should be on a local or mounted drive.");
+ }
+ dataSetDirectoryProvider = dataStoreService.getDataSetDirectoryProvider();
+ File storeRoot = dataSetDirectoryProvider.getStoreRoot();
+ properties.setProperty(AbstractArchiverProcessingPlugin.SYNCHRONIZE_ARCHIVE, "false");
+ archiver = new Archiver(properties, storeRoot, fileOperationManager);
+ }
+
+ @Override
+ public IPostRegistrationTaskExecutor createExecutor(String dataSetCode)
+ {
+ return new ArchivingExecutor(dataSetCode, false, service, archiver, dataSetDirectoryProvider,
+ hierarchicalContentProvider);
+ }
+
+ private static final class Archiver extends RsyncArchiver
+ {
+
+ private static final long serialVersionUID = 1L;
+
+ Archiver(Properties properties, File storeRoot,
+ IDataSetFileOperationsManager fileOperationsManager)
+ {
+ super(properties, storeRoot, fileOperationsManager, RsyncArchiver.DeleteAction.DELETE,
+ ChecksumVerificationCondition.IF_AVAILABLE);
+ setStatusUpdater(new IDataSetStatusUpdater()
+ {
+ @Override
+ public void update(List<String> dataSetCodes, DataSetArchivingStatus status,
+ boolean presentInArchive)
+ {
+ }
+ });
+ }
+
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
index bac63dbcba8bada5d75f64db0f12ac84d8237340..504926852e11d679f708045bc3632fc841bd7f1e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
@@ -79,7 +79,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
@Private
public static final String SHARE_FINDER_KEY = "share-finder";
- @Private static final String SYNCHRONIZE_ARCHIVE = "synchronize-archive";
+ public static final String SYNCHRONIZE_ARCHIVE = "synchronize-archive";
private final IStatusChecker archivePrerequisiteOrNull;
@@ -547,8 +547,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
return service;
}
- @Private
- void setStatusUpdater(IDataSetStatusUpdater statusUpdater)
+ public void setStatusUpdater(IDataSetStatusUpdater statusUpdater)
{
this.statusUpdater = statusUpdater;
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
index d33cf766d11cf8a9bb374cb3f99d507f78488b76..c5214a04721ab31dba94f02f9defca42353e66bc 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
@@ -61,6 +61,36 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
private static final long serialVersionUID = 1L;
+ public enum ChecksumVerificationCondition
+ {
+ NO()
+ {
+ @Override
+ boolean verifyChecksum(IHierarchicalContentNode node)
+ {
+ return false;
+ }
+ },
+ YES()
+ {
+ @Override
+ boolean verifyChecksum(IHierarchicalContentNode node)
+ {
+ return true;
+ }
+ },
+ IF_AVAILABLE()
+ {
+ @Override
+ boolean verifyChecksum(IHierarchicalContentNode node)
+ {
+ return node.isChecksumCRC32Precalculated();
+ }
+ };
+
+ abstract boolean verifyChecksum(IHierarchicalContentNode node);
+ }
+
private static final Comparator<IHierarchicalContentNode> NODE_COMPARATOR =
new Comparator<IHierarchicalContentNode>()
{
@@ -71,7 +101,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
}
};
- private enum DeleteAction
+ public enum DeleteAction
{
DELETE(Operation.DELETE_FROM_ARCHIVE)
{
@@ -109,7 +139,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
private final DeleteAction deleteAction;
- private final boolean verifyChecksums;
+ private final ChecksumVerificationCondition checksumVerificationCondition;
public RsyncArchiver(Properties properties, File storeRoot)
{
@@ -117,22 +147,25 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
new RsyncArchiveCopierFactory(), new SshCommandExecutorFactory()));
}
- @Private
- RsyncArchiver(Properties properties, File storeRoot,
+ @Private RsyncArchiver(Properties properties, File storeRoot,
IDataSetFileOperationsManager fileOperationsManager)
{
- super(properties, storeRoot, null, null);
- this.fileOperationsManager = fileOperationsManager;
- if (PropertyUtils.getBoolean(properties, ONLY_MARK_AS_DELETED_KEY, true))
- {
- deleteAction = DeleteAction.MARK_AS_DELETED;
- } else
- {
- deleteAction = DeleteAction.DELETE;
- }
+ this(properties, storeRoot, fileOperationsManager, PropertyUtils.getBoolean(properties,
+ ONLY_MARK_AS_DELETED_KEY, true) ? DeleteAction.MARK_AS_DELETED
+ : DeleteAction.DELETE, PropertyUtils.getBoolean(properties, VERIFY_CHECKSUMS_KEY,
+ true) ? ChecksumVerificationCondition.YES : ChecksumVerificationCondition.NO);
- verifyChecksums = PropertyUtils.getBoolean(properties, VERIFY_CHECKSUMS_KEY, true);
+ }
+ public RsyncArchiver(Properties properties, File storeRoot,
+ IDataSetFileOperationsManager fileOperationsManager, DeleteAction deleteAction,
+ ChecksumVerificationCondition checksumVerificationCondition)
+ {
+ super(properties, storeRoot, null, null);
+ this.fileOperationsManager = fileOperationsManager;
+ this.deleteAction = deleteAction;
+ this.checksumVerificationCondition = checksumVerificationCondition;
+
}
@Override
@@ -178,7 +211,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
IHierarchicalContentNode root = content.getRootNode();
IHierarchicalContentNode archivedRoot = archivedContent.getRootNode();
- status = checkHierarchySizeAndChecksums(root, archivedRoot, verifyChecksums);
+ status = checkHierarchySizeAndChecksums(root, archivedRoot, checksumVerificationCondition);
} finally
{
FileUtils.deleteQuietly(temp);
@@ -192,7 +225,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
@Private
static Status checkHierarchySizeAndChecksums(IHierarchicalContentNode node,
- IHierarchicalContentNode retrievedNode, boolean verifyChecksums)
+ IHierarchicalContentNode retrievedNode, ChecksumVerificationCondition checksumVerificationCondition)
{
String relativePath = node.getRelativePath();
String relativePathOfRetrieved = retrievedNode.getRelativePath();
@@ -226,7 +259,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
{
Status status =
checkHierarchySizeAndChecksums(childNodes.get(i),
- childNodesOfRetrieved.get(i), verifyChecksums);
+ childNodesOfRetrieved.get(i), checksumVerificationCondition);
if (status.isError())
{
return status;
@@ -241,7 +274,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
return Status.createError("The file '" + relativePath + "' has in the store "
+ fileLength + " bytes but " + fileLengthOfRetrieved + " in the archive.");
}
- if (verifyChecksums)
+ if (checksumVerificationCondition.verifyChecksum(node))
{
long checksum = node.getChecksumCRC32();
long checksumOfRetrieved = retrievedNode.getChecksumCRC32();
@@ -255,7 +288,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
}
return Status.OK;
}
-
+
private static List<IHierarchicalContentNode> getChildNodes(IHierarchicalContentNode node)
{
List<IHierarchicalContentNode> childNodes = node.getChildNodes();
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
index 6466243798f6dd4238a0e22a6247179a1d7cf8bc..6c49eb627260fb03c925c90048a12695de509acb 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiverTest.java
@@ -478,8 +478,10 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
IHierarchicalContentNode root2 =
new MockContent(":0:0", "a/:0:0", "a/f2.txt:15:13", "a/f1.txt:5:-3", "r.txt:7:17")
.getRootNode();
- assertEquals("OK", RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true)
- .toString());
+ assertEquals(
+ "OK",
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2,
+ RsyncArchiver.ChecksumVerificationCondition.YES).toString());
}
@Test
@@ -489,9 +491,11 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
new MockContent(":0:0", "a/:0:0", "a/f1.txt:5:-3").getRootNode();
IHierarchicalContentNode root2 =
new MockContent(":0:0", "a/:0:0", "a/f3.txt:15:13").getRootNode();
- assertEquals("ERROR: \"Different paths: Path in the store is 'a/f1.txt' "
- + "and in the archive 'a/f3.txt'.\"",
- RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
+ assertEquals(
+ "ERROR: \"Different paths: Path in the store is 'a/f1.txt' "
+ + "and in the archive 'a/f3.txt'.\"",
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2,
+ RsyncArchiver.ChecksumVerificationCondition.YES).toString());
}
@Test
@@ -499,10 +503,12 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
{
IHierarchicalContentNode root1 = new MockContent(":0:0", "a/:0:0").getRootNode();
IHierarchicalContentNode root2 = new MockContent(":0:0", "a:1:2").getRootNode();
- assertEquals("ERROR: \"The path 'a' should be in store and archive either "
- + "both directories or files but not mixed: In the store it is a directory "
- + "but in the archive it is a file.\"", RsyncArchiver
- .checkHierarchySizeAndChecksums(root1, root2, true).toString());
+ assertEquals(
+ "ERROR: \"The path 'a' should be in store and archive either "
+ + "both directories or files but not mixed: In the store it is a directory "
+ + "but in the archive it is a file.\"",
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2,
+ RsyncArchiver.ChecksumVerificationCondition.YES).toString());
}
@Test
@@ -512,8 +518,10 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
new MockContent(":0:0", "a/:0:0", "a/f1.txt:5:-3", "a/f2.txt:15:13").getRootNode();
IHierarchicalContentNode root2 =
new MockContent(":0:0", "a/:0:0", "a/f2.txt:15:13").getRootNode();
- assertEquals("ERROR: \"The directory 'a' has in the store 2 files but 1 in the archive.\"",
- RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
+ assertEquals(
+ "ERROR: \"The directory 'a' has in the store 2 files but 1 in the archive.\"",
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2,
+ RsyncArchiver.ChecksumVerificationCondition.YES).toString());
}
@Test
@@ -521,8 +529,10 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
{
IHierarchicalContentNode root1 = new MockContent(":0:0", "r.txt:7:17").getRootNode();
IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:9:17").getRootNode();
- assertEquals("ERROR: \"The file 'r.txt' has in the store 7 bytes but 9 in the archive.\"",
- RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
+ assertEquals(
+ "ERROR: \"The file 'r.txt' has in the store 7 bytes but 9 in the archive.\"",
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2,
+ RsyncArchiver.ChecksumVerificationCondition.YES).toString());
}
@Test
@@ -530,17 +540,21 @@ public class RsyncArchiverTest extends AbstractFileSystemTestCase
{
IHierarchicalContentNode root1 = new MockContent(":0:0", "r.txt:7:17").getRootNode();
IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:7:18").getRootNode();
- assertEquals("ERROR: \"The file 'r.txt' has in the store the checksum 00000017 "
- + "but 00000018 in the archive.\"",
- RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, true).toString());
+ assertEquals(
+ "ERROR: \"The file 'r.txt' has in the store the checksum 00000017 "
+ + "but 00000018 in the archive.\"",
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2,
+ RsyncArchiver.ChecksumVerificationCondition.YES).toString());
}
public void testCheckHierarchySizeAndChecksumsWrongChecksumAreNotChecked()
{
IHierarchicalContentNode root1 = new MockContent(":0:0", "r.txt:7:17").getRootNode();
IHierarchicalContentNode root2 = new MockContent(":0:0", "r.txt:7:18").getRootNode();
- assertEquals("OK", RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2, false)
- .toString());
+ assertEquals(
+ "OK",
+ RsyncArchiver.checkHierarchySizeAndChecksums(root1, root2,
+ RsyncArchiver.ChecksumVerificationCondition.NO).toString());
}
private static final class MockNode implements IHierarchicalContentNode
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/postregistration/ScreeningHdf5PostRegistrationTask.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/postregistration/ScreeningHdf5PostRegistrationTask.java
index 731ff12372f63e5a7b929d0f26f3ae97324c751f..6630ba70a5a212832a27a2574b589b3632dbbd5f 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/postregistration/ScreeningHdf5PostRegistrationTask.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/postregistration/ScreeningHdf5PostRegistrationTask.java
@@ -27,7 +27,6 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.etlserver.postregistration.DummyPostRegistrationTaskExecutor;
import ch.systemsx.cisd.etlserver.postregistration.Hdf5CompressingPostRegistrationTask;
import ch.systemsx.cisd.etlserver.postregistration.IPostRegistrationTaskExecutor;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -52,12 +51,8 @@ public class ScreeningHdf5PostRegistrationTask extends Hdf5CompressingPostRegist
}
@Override
- public IPostRegistrationTaskExecutor createExecutor(String dataSetCode, boolean container)
+ public IPostRegistrationTaskExecutor createExecutor(String dataSetCode)
{
- if (container)
- {
- return DummyPostRegistrationTaskExecutor.INSTANCE;
- }
return new ScreeningExecutor(dataSetCode, properties);
}