From 6de92e935f132b6af737ccd56845af8e5102123f Mon Sep 17 00:00:00 2001
From: kaloyane <kaloyane>
Date: Tue, 15 Mar 2011 08:27:25 +0000
Subject: [PATCH] [LMS-2087] Eager archiving (part 6 - initial implementation
for deleteFromArchive)
SVN: 20320
---
...DeletionPostProcessingMaintenanceTask.java | 127 +++++++++
.../DeleteFromArchiveMaintenanceTask.java | 113 ++++++++
.../DeleteFromExternalDBMaintenanceTask.java | 259 ++++++++++++++++++
.../dss/generic/server/DataStoreService.java | 4 +-
.../server/EncapsulatedOpenBISService.java | 7 +-
.../OpenBISAuthenticationInterceptor.java | 2 +-
.../server/plugins/demo/DemoArchiver.java | 18 +-
.../AbstractArchiverProcessingPlugin.java | 25 +-
.../plugins/standard/RsyncArchiver.java | 10 +-
.../plugins/standard/RsyncDataSetCopier.java | 10 +-
.../server/plugins/tasks/IArchiverPlugin.java | 4 +-
.../plugins/tasks/PluginTaskProviders.java | 2 +-
.../shared/IEncapsulatedOpenBISService.java | 7 +-
.../openbis/generic/server/CommonServer.java | 5 +-
.../openbis/generic/server/ETLService.java | 6 +-
.../generic/server/ETLServiceLogger.java | 6 +-
.../server/business/bo/ExternalDataTable.java | 4 +-
.../generic/server/dataaccess/IEventDAO.java | 6 +-
.../server/dataaccess/db/EventDAO.java | 13 +-
.../generic/shared/IETLLIMSService.java | 3 +-
.../shared/basic/dto/DeletedDataSet.java | 43 ++-
.../shared/dto/DatasetDescription.java | 15 +
.../server/dataaccess/db/EventDAOTest.java | 37 ++-
.../shared/IETLLIMSService.java.expected | 3 +-
.../cisd/yeastx/etl/MLArchiverTask.java | 11 +-
.../yeastx/etl/MetabolDatabaseUpdater.java | 4 +-
26 files changed, 665 insertions(+), 79 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/AbstractDataSetDeletionPostProcessingMaintenanceTask.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DeleteFromArchiveMaintenanceTask.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DeleteFromExternalDBMaintenanceTask.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/AbstractDataSetDeletionPostProcessingMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/AbstractDataSetDeletionPostProcessingMaintenanceTask.java
new file mode 100644
index 00000000000..6f63864adb6
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/AbstractDataSetDeletionPostProcessingMaintenanceTask.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.lang.time.DateUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
+
+/**
+ * Maintenance task which executes after a data set has been deleted in openBIS. Example use cases
+ * for it include purging data set archives or/and external databases.
+ *
+ * @author Kaloyan Enimanev
+ */
+public abstract class AbstractDataSetDeletionPostProcessingMaintenanceTask implements
+ IMaintenanceTask
+{
+
+ protected static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, AbstractDataSetDeletionPostProcessingMaintenanceTask.class);
+
+ protected static final String DELAY_AFTER_DELETION = "delay-after-deletion";
+
+ protected final IEncapsulatedOpenBISService openBISService;
+
+ protected long delayAfterDeletion;
+
+ protected abstract Long getLastSeenEventId();
+
+ protected abstract void updateLastSeenEventId(Long eventId);
+
+ protected abstract void execute(List<DeletedDataSet> datasetCodes);
+
+ public AbstractDataSetDeletionPostProcessingMaintenanceTask()
+ {
+ LogInitializer.init();
+ openBISService = ServiceProvider.getOpenBISService();
+ }
+
+ public void setUp(String pluginName, Properties properties)
+ {
+ int delayInMinutes = PropertyUtils.getInt(properties, DELAY_AFTER_DELETION, 0);
+
+ delayAfterDeletion = delayInMinutes * DateUtils.MILLIS_PER_MINUTE;
+ }
+
+ public void execute()
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug("Synchronizing data set data deletion");
+ }
+ try
+ {
+ Long lastSeenEventId = getLastSeenEventId();
+ List<DeletedDataSet> deletedDataSets =
+ openBISService.listDeletedDataSets(lastSeenEventId, computeMaxDeletionDate());
+ if (deletedDataSets.size() > 0)
+ {
+
+ long t0 = System.currentTimeMillis();
+
+ execute(deletedDataSets);
+ updateLastSeenEventId(deletedDataSets, lastSeenEventId);
+
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Data set deletion post-processing task took "
+ + ((System.currentTimeMillis() - t0 + 500) / 1000) + " seconds.");
+ }
+ }
+ } catch (Throwable t)
+ {
+ operationLog.error("Failed to process data-set deletion info :", t);
+ }
+ }
+
+ private Date computeMaxDeletionDate()
+ {
+ long now = System.currentTimeMillis();
+ long maxDeletionTimestamp = now - delayAfterDeletion;
+ return new Date(maxDeletionTimestamp);
+ }
+
+ private void updateLastSeenEventId(List<DeletedDataSet> deleted, Long lastSeenEventIdOrNull)
+ {
+ Long maxEventId = lastSeenEventIdOrNull;
+ for (DeletedDataSet dds : deleted)
+ {
+ long eventId = dds.getEventId();
+ if (maxEventId == null || eventId > maxEventId)
+ {
+ maxEventId = eventId;
+ }
+ }
+ if (lastSeenEventIdOrNull == null || maxEventId > lastSeenEventIdOrNull)
+ {
+ updateLastSeenEventId(maxEventId);
+ }
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DeleteFromArchiveMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DeleteFromArchiveMaintenanceTask.java
new file mode 100644
index 00000000000..693109c933d
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DeleteFromArchiveMaintenanceTask.java
@@ -0,0 +1,113 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.io.File;
+import java.util.List;
+import java.util.Properties;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.collections.CollectionUtils;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ArchiverPluginFactory;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IArchiverPlugin;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.PluginTaskProviders;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
+
+/**
+ * Maintenance task that removes data sets from the data store archive after they have been deleted
+ * in openBIS.
+ *
+ * @author Kaloyan Enimanev
+ */
+public class DeleteFromArchiveMaintenanceTask extends
+ AbstractDataSetDeletionPostProcessingMaintenanceTask
+{
+
+ // a file name to store the last seen event id
+ private static final String STATUS_FILENAME = "status-filename";
+
+ private File lastSeenEventIdFile;
+
+ @Override
+ public void setUp(String pluginName, Properties properties)
+ {
+ super.setUp(pluginName, properties);
+ String eventIdFileName = PropertyUtils.getMandatoryProperty(properties, STATUS_FILENAME);
+ lastSeenEventIdFile = new File(eventIdFileName);
+ }
+
+ @Override
+ protected Long getLastSeenEventId()
+ {
+ Long result = null;
+ if (lastSeenEventIdFile.exists())
+ {
+ try
+ {
+ String statusFileContent = FileUtilities.loadToString(lastSeenEventIdFile);
+ result = Long.parseLong(statusFileContent);
+ } catch (Exception ex)
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug("Cannot load last seen event id from file :"
+ + lastSeenEventIdFile, ex);
+ }
+ }
+ }
+ return result;
+ }
+
+ @Override
+ protected void updateLastSeenEventId(Long newLastSeenEventId)
+ {
+ try
+ {
+ // create a temporary file (not an atomic operation)
+ File tmpFile =
+ File.createTempFile(lastSeenEventIdFile.getName(), "tmp",
+ lastSeenEventIdFile.getParentFile());
+ String fileContent = String.valueOf(newLastSeenEventId);
+ FileUtilities.writeToFile(tmpFile, fileContent);
+
+ // move operation (should be atomic)
+ tmpFile.renameTo(lastSeenEventIdFile);
+ } catch (Exception ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ @Override
+ protected void execute(List<DeletedDataSet> datasets)
+ {
+ PluginTaskProviders provider = PluginTaskProviders.create();
+ ArchiverPluginFactory archiverFactory = provider.getArchiverPluginFactory();
+ IArchiverPlugin archiver = archiverFactory.createInstance(provider.getStoreRoot());
+
+ archiver.deleteFromArchive(datasets);
+
+ String logMessage =
+ String.format("Deleted %s dataset from archive: '%s'", datasets.size(),
+ CollectionUtils.abbreviate(datasets, 10));
+ operationLog.info(logMessage);
+
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DeleteFromExternalDBMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DeleteFromExternalDBMaintenanceTask.java
new file mode 100644
index 00000000000..11670c2c242
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/DeleteFromExternalDBMaintenanceTask.java
@@ -0,0 +1,259 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.List;
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.lang.StringEscapeUtils;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
+
+/**
+ * Maintenance task deleting from a custom-specific database data sets which have been deleted from
+ * openbis.
+ *
+ * @author Izabela Adamczyk
+ */
+public class DeleteFromExternalDBMaintenanceTask extends
+ AbstractDataSetDeletionPostProcessingMaintenanceTask
+{
+
+ private static final String DEFAULT_DATA_SET_PERM_ID = "PERM_ID";
+
+ private static final String DATA_SET_PERM_ID_KEY = "data-set-perm-id";
+
+ private static final String DATA_SET_TABLE_NAME_KEY = "data-set-table-name";
+
+ private static final String DEFAULT_DATA_SET_TABLE_NAME = "data_sets";
+
+ private static final String LAST_SEEN_EVENT_ID_COLUMN_KEY = "last-seen-event-id-column";
+
+ private static final String SYNCHRONIZATION_TABLE_KEY = "synchronization-table";
+
+ private static final String DEFAULT_SYNCHRONIZATION_TABLE = "EVENTS";
+
+ private static final String DEFAULT_LAST_SEEN_EVENT_ID = "LAST_SEEN_DELETION_EVENT_ID";
+
+ private DataSource dataSource;
+
+ private String synchronizationTable;
+
+ private String lastSeenEventID;
+
+ private String dataSetTableName;
+
+ private String permIDColumn;
+
+ private Connection connection;
+
+ @Override
+ public void setUp(String pluginName, Properties properties)
+ {
+ super.setUp(pluginName, properties);
+ synchronizationTable =
+ properties.getProperty(SYNCHRONIZATION_TABLE_KEY, DEFAULT_SYNCHRONIZATION_TABLE);
+ lastSeenEventID =
+ properties.getProperty(LAST_SEEN_EVENT_ID_COLUMN_KEY, DEFAULT_LAST_SEEN_EVENT_ID);
+ dataSetTableName =
+ properties.getProperty(DATA_SET_TABLE_NAME_KEY, DEFAULT_DATA_SET_TABLE_NAME);
+ permIDColumn = properties.getProperty(DATA_SET_PERM_ID_KEY, DEFAULT_DATA_SET_PERM_ID);
+ this.dataSource = ServiceProvider.getDataSourceProvider().getDataSource(properties);
+ checkDatabaseConnection();
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Plugin initialized: " + pluginName);
+ }
+ }
+
+ /**
+ * method overriden to create an underlying database connection.
+ */
+ @Override
+ public void execute()
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug("Synchronizing data set information");
+ }
+
+ connection = null;
+ try
+ {
+ connection = createConnection();
+ super.execute();
+
+ } catch (SQLException sqlEx)
+ {
+ operationLog.error(sqlEx);
+ } finally
+ {
+ closeConnection(connection);
+ }
+ }
+
+ @Override
+ protected Long getLastSeenEventId()
+ {
+ try
+ {
+ return tryGetPreviousLastSeenEventId(connection);
+ } catch (SQLException sqlEx)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(sqlEx);
+ }
+ }
+
+ @Override
+ protected void updateLastSeenEventId(Long newLastSeenEventId)
+ {
+ try
+ {
+ executeSql("delete from " + synchronizationTable);
+ executeSql("INSERT INTO " + synchronizationTable + " (" + lastSeenEventID
+ + ") VALUES('" + newLastSeenEventId + "')");
+ } catch (SQLException sqlEx)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(sqlEx);
+ }
+
+ }
+
+ @Override
+ protected void execute(List<DeletedDataSet> datasets)
+ {
+ try
+ {
+ boolean autoCommit = connection.getAutoCommit();
+ connection.setAutoCommit(false);
+ long t0 = System.currentTimeMillis();
+ deleteDatasets(datasets);
+ connection.commit();
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Synchronization task took "
+ + ((System.currentTimeMillis() - t0 + 500) / 1000) + " seconds.");
+ }
+ connection.setAutoCommit(autoCommit);
+ } catch (SQLException sqlEx)
+ {
+ operationLog.error(sqlEx);
+ } finally
+ {
+ closeConnection(connection);
+ }
+
+ }
+
+ private void checkDatabaseConnection()
+ {
+ Connection c = null;
+ try
+ {
+ c = createConnection();
+ tryGetPreviousLastSeenEventId(c);
+ } catch (SQLException sqlEx)
+ {
+ throw new ConfigurationFailureException("Initialization failed", sqlEx);
+ } finally
+ {
+ closeConnection(c);
+ }
+ }
+
+ private void closeConnection(Connection connectionOrNull)
+ {
+ if (connectionOrNull != null)
+ {
+ try
+ {
+ connectionOrNull.close();
+ } catch (SQLException ex)
+ {
+ // suppress this exception
+ operationLog.error(ex);
+ }
+ }
+ }
+
+ private Connection createConnection() throws SQLException
+ {
+ return dataSource.getConnection();
+ }
+
+ private void deleteDatasets(List<DeletedDataSet> deletedDataSets)
+ throws SQLException
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info(String
+ .format("Synchronizing deletions of %d datasets with the database.",
+ deletedDataSets.size()));
+ }
+ connection.createStatement().execute(
+ String.format("DELETE FROM " + dataSetTableName + " WHERE " + permIDColumn
+ + " IN (%s)", joinIds(deletedDataSets)));
+ }
+
+ private void executeSql(String sql) throws SQLException
+ {
+ PreparedStatement statement = connection.prepareStatement(sql);
+ statement.executeUpdate();
+ }
+
+ private String joinIds(List<DeletedDataSet> deletedDatasetCodes)
+ {
+ StringBuilder sb = new StringBuilder();
+ for (DeletedDataSet dds : deletedDatasetCodes)
+ {
+ if (sb.length() != 0)
+ {
+ sb.append(", ");
+ }
+ sb.append("'" + StringEscapeUtils.escapeSql(dds.getIdentifier()) + "'");
+ }
+ String ids = sb.toString();
+ return ids;
+ }
+
+ private long tryGetPreviousLastSeenEventId(Connection c) throws SQLException
+ {
+ Long maxLastSeenEventId = null;
+ ResultSet result =
+ c.createStatement().executeQuery(
+ "SELECT MAX(" + lastSeenEventID + ") AS " + lastSeenEventID + " FROM "
+ + synchronizationTable);
+ while (result.next())
+ {
+ long newLastSeenEventId = result.getLong(lastSeenEventID);
+ if (maxLastSeenEventId == null || maxLastSeenEventId < newLastSeenEventId)
+ {
+ maxLastSeenEventId = newLastSeenEventId;
+ }
+ }
+ return maxLastSeenEventId;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java
index ce7f585d66f..8ff60ae1aae 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java
@@ -61,7 +61,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
import ch.systemsx.cisd.openbis.generic.shared.util.UuidUtil;
/**
- * Implementation of {@link IDataStoreService} which will be accessed remotely by the opneBIS
+ * Implementation of {@link IDataStoreService} which will be accessed remotely by the openBIS
* server.
*
* @author Franz-Josef Elmer
@@ -353,7 +353,7 @@ public class DataStoreService extends AbstractServiceWithLogger<IDataStoreServic
private IArchiverPlugin createArchiver()
{
- ArchiverPluginFactory factory = pluginTaskParameters.getArchiverTaskFactory();
+ ArchiverPluginFactory factory = pluginTaskParameters.getArchiverPluginFactory();
return factory.createInstance(storeRoot);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
index 6aed4904365..7c634a3036e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/EncapsulatedOpenBISService.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.generic.server;
import java.util.Collection;
+import java.util.Date;
import java.util.List;
import org.apache.log4j.Logger;
@@ -421,9 +422,11 @@ public final class EncapsulatedOpenBISService implements IEncapsulatedOpenBISSer
criteria);
}
- public List<DeletedDataSet> listDeletedDataSets(Long lastSeenDeletionEventIdOrNull)
+ public List<DeletedDataSet> listDeletedDataSets(Long lastSeenDeletionEventIdOrNull,
+ Date maxDeletionDataOrNull)
{
- return service.listDeletedDataSets(session.getToken(), lastSeenDeletionEventIdOrNull);
+ return service.listDeletedDataSets(session.getToken(), lastSeenDeletionEventIdOrNull,
+ maxDeletionDataOrNull);
}
public void archiveDataSets(List<String> dataSetCodes, boolean removeFromDataStore)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/openbisauth/OpenBISAuthenticationInterceptor.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/openbisauth/OpenBISAuthenticationInterceptor.java
index 7824ce02d2a..1df41c35a68 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/openbisauth/OpenBISAuthenticationInterceptor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/openbisauth/OpenBISAuthenticationInterceptor.java
@@ -95,7 +95,7 @@ public class OpenBISAuthenticationInterceptor implements MethodInterceptor
this.service = service;
this.pluginTaskDescriptions = pluginTaskParameters.getPluginTaskDescriptions();
this.archiverConfigured =
- pluginTaskParameters.getArchiverTaskFactory().isArchiverConfigured();
+ pluginTaskParameters.getArchiverPluginFactory().isArchiverConfigured();
this.sessionHolder = sessionHolder;
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
index 259e3a3f01e..ad74609b84a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoArchiver.java
@@ -27,6 +27,7 @@ import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.AbstractArchiverProcessingPlugin;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ArchiverTaskContext;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -48,7 +49,7 @@ public class DemoArchiver extends AbstractArchiverProcessingPlugin
ArchiverTaskContext context) throws UserFailureException
{
System.out.println("DemoArchiver - Archived: " + datasets);
- archiveContents.addAll(extractCodes(datasets));
+ archiveContents.addAll(DatasetDescription.extractCodes(datasets));
return createStatuses(Status.OK, datasets, Operation.ARCHIVE);
}
@@ -69,12 +70,17 @@ public class DemoArchiver extends AbstractArchiverProcessingPlugin
}
@Override
- protected DatasetProcessingStatuses doDeleteFromArchive(List<DatasetDescription> datasets,
- ArchiverTaskContext context) throws UserFailureException
+ public DatasetProcessingStatuses doDeleteFromArchive(List<DeletedDataSet> dataSets)
{
- archiveContents.addAll(extractCodes(datasets));
- System.out.println("DemoArchiver - deleteFromArchive: " + datasets);
- return createStatuses(Status.OK, datasets, Operation.DELETE_FROM_ARCHIVE);
+ List<String> datasetCodes = DeletedDataSet.extractDataSetCodes(dataSets);
+ archiveContents.addAll(datasetCodes);
+ System.out.println("DemoArchiver - deleteFromArchive: " + datasetCodes);
+ DatasetProcessingStatuses statuses = new DatasetProcessingStatuses();
+ for (String dataset : datasetCodes)
+ {
+ statuses.addResult(dataset, Status.OK, Operation.DELETE_FROM_ARCHIVE);
+ }
+ return statuses;
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
index a44a92d4c98..8bed6e58a89 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
@@ -37,6 +37,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.QueueingDataSetStatusUpdaterS
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetCodesWithStatus;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -76,13 +77,12 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
*/
abstract protected DatasetProcessingStatuses doUnarchive(List<DatasetDescription> datasets,
ArchiverTaskContext context);
-
+
/**
- * NOTE: this method is not allowed to throw exception as this will leave data sets in the
- * openBIS database with an inconsistent status.
+ * deletes data sets from archive. At the time when this method is invoken the data sets do not
+ * exist in the openBIS database.
*/
- abstract protected DatasetProcessingStatuses doDeleteFromArchive(
- List<DatasetDescription> datasets, ArchiverTaskContext context);
+ abstract protected DatasetProcessingStatuses doDeleteFromArchive(List<DeletedDataSet> datasets);
/**
* @return <code>true</code> if the dataset is present in the archive, <code>false</code>
@@ -174,10 +174,9 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
return statuses.getProcessingStatus();
}
- public ProcessingStatus deleteFromArchive(List<DatasetDescription> datasets,
- ArchiverTaskContext context)
+ public ProcessingStatus deleteFromArchive(List<DeletedDataSet> datasets)
{
- DatasetProcessingStatuses status = doDeleteFromArchive(datasets, context);
+ DatasetProcessingStatuses status = doDeleteFromArchive(datasets);
return status != null ? status.getProcessingStatus() : null;
}
@@ -305,16 +304,6 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
return statuses;
}
- protected List<String> extractCodes(List<DatasetDescription> dataSets)
- {
- List<String> result = new ArrayList<String>();
- for (DatasetDescription description : dataSets)
- {
- result.add(description.getDatasetCode());
- }
- return result;
- }
-
private static void asyncUpdateStatuses(List<String> dataSetCodes,
DataSetArchivingStatus newStatus, boolean presentInArchive)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
index 147908ddc27..f24028350e1 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
@@ -25,6 +25,7 @@ import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ArchiverTaskContext;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -100,17 +101,16 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
}
@Override
- protected DatasetProcessingStatuses doDeleteFromArchive(List<DatasetDescription> datasets,
- ArchiverTaskContext context)
+ protected DatasetProcessingStatuses doDeleteFromArchive(List<DeletedDataSet> datasets)
{
initIfNecessary();
// no need to lock - this is processing task
DatasetProcessingStatuses statuses = new DatasetProcessingStatuses();
- for (DatasetDescription dataset : datasets)
+ for (DeletedDataSet dataset : datasets)
{
Status status = doDeleteFromArchive(dataset);
- statuses.addResult(dataset.getDatasetCode(), status, Operation.DELETE_FROM_ARCHIVE);
+ statuses.addResult(dataset.getIdentifier(), status, Operation.DELETE_FROM_ARCHIVE);
}
return statuses;
@@ -145,7 +145,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
return copier.retrieveFromDestination(originalData, dataset);
}
- private Status doDeleteFromArchive(DatasetDescription dataset)
+ private Status doDeleteFromArchive(DeletedDataSet dataset)
{
return copier.deleteFromDestination(dataset);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopier.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopier.java
index ad38c8149ff..830af990768 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopier.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopier.java
@@ -37,6 +37,7 @@ import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.openbis.dss.generic.server.IDataSetFileOperationsExecutor;
import ch.systemsx.cisd.openbis.dss.generic.server.LocalDataSetFileOperationsExcecutor;
import ch.systemsx.cisd.openbis.dss.generic.server.RemoteDataSetFileOperationsExecutor;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -152,18 +153,21 @@ public class RsyncDataSetCopier // TODO rename to DataSetFileOperationsManager
* Deletes specified datases's data from the destination specified in constructor. The path at
* the destination is defined by original location of the data set.
*/
- public Status deleteFromDestination(DatasetDescription dataset)
+ public Status deleteFromDestination(DeletedDataSet dataset)
{
try
{
- File destinationFolder = new File(destination, dataset.getDataSetLocation());
+ // TODO KE: Piotr, here we might have "invalid" values for location
+ // coming from the old-style description contents. We could only execute the following
+ // logic if dataset.getLocation() != dataset.getIdentifier() ?
+ File destinationFolder = new File(destination, dataset.getLocation());
BooleanStatus destinationExists = destinationExists(destinationFolder);
if (destinationExists.isSuccess())
{
executor.deleteFolder(destinationFolder);
} else
{
- operationLog.info("Data of data set '" + dataset.getDatasetCode()
+ operationLog.info("Data of data set '" + dataset.getIdentifier()
+ "' don't exist in the destination '" + destinationFolder.getPath()
+ "'. There is nothing to delete.");
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/IArchiverPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/IArchiverPlugin.java
index 7101dce6397..267efb1eea9 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/IArchiverPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/IArchiverPlugin.java
@@ -19,6 +19,7 @@ package ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks;
import java.io.Serializable;
import java.util.List;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -54,6 +55,5 @@ public interface IArchiverPlugin extends Serializable
* @returns {@link ProcessingStatus} containing the deletion statuses for all data sets or null
* if processing succeeded for all datasets and no additional information is provided.
*/
- ProcessingStatus deleteFromArchive(List<DatasetDescription> datasets,
- ArchiverTaskContext context);
+ ProcessingStatus deleteFromArchive(List<DeletedDataSet> datasets);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/PluginTaskProviders.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/PluginTaskProviders.java
index edc1b860e29..41dcd181f10 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/PluginTaskProviders.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/PluginTaskProviders.java
@@ -92,7 +92,7 @@ public class PluginTaskProviders
return processingPlugins;
}
- public ArchiverPluginFactory getArchiverTaskFactory()
+ public ArchiverPluginFactory getArchiverPluginFactory()
{
return archiverTaskFactory;
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
index dc0ce6cbe57..d8e410257e2 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IEncapsulatedOpenBISService.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.generic.shared;
import java.util.Collection;
+import java.util.Date;
import java.util.List;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
@@ -291,9 +292,13 @@ public interface IEncapsulatedOpenBISService
/**
* List data sets deleted after the last seen deletion event. If event id is null all deleted
* datasets will be returned.
+ *
+ * @param maxDeletionDateOrNull when specified only lists data sets that have been deleted
+ * before it.
*/
@ManagedAuthentication
- public List<DeletedDataSet> listDeletedDataSets(Long lastSeenDeletionEventIdOrNull);
+ public List<DeletedDataSet> listDeletedDataSets(Long lastSeenDeletionEventIdOrNull,
+ Date maxDeletionDateOrNull);
/**
* Updates specified properties of given data set.
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServer.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServer.java
index 6183855096f..1a4d3fc1c0a 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServer.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServer.java
@@ -2002,10 +2002,11 @@ public final class CommonServer extends AbstractCommonServer<ICommonServerForInt
}
public List<DeletedDataSet> listDeletedDataSets(String sessionToken,
- Long lastSeenDeletionEventIdOrNull)
+ Long lastSeenDeletionEventIdOrNull, Date maxDeletionDataOrNull)
{
checkSession(sessionToken);
- return getDAOFactory().getEventDAO().listDeletedDataSets(lastSeenDeletionEventIdOrNull);
+ return getDAOFactory().getEventDAO().listDeletedDataSets(lastSeenDeletionEventIdOrNull,
+ maxDeletionDataOrNull);
}
// --- grid custom filters and columns
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
index 8d0243a6d26..7449565102e 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
@@ -19,6 +19,7 @@ package ch.systemsx.cisd.openbis.generic.server;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
+import java.util.Date;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -775,10 +776,11 @@ public class ETLService extends AbstractCommonServer<IETLService> implements IET
}
public List<DeletedDataSet> listDeletedDataSets(String sessionToken,
- Long lastSeenDeletionEventIdOrNull)
+ Long lastSeenDeletionEventIdOrNull, Date maxDeletionDataOrNull)
{
checkSession(sessionToken);
- return getDAOFactory().getEventDAO().listDeletedDataSets(lastSeenDeletionEventIdOrNull);
+ return getDAOFactory().getEventDAO().listDeletedDataSets(lastSeenDeletionEventIdOrNull,
+ maxDeletionDataOrNull);
}
public ExternalData tryGetDataSetForServer(String sessionToken, String dataSetCode)
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLServiceLogger.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLServiceLogger.java
index 1d1cb22c247..ffc2041a547 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLServiceLogger.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLServiceLogger.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.generic.server;
import java.util.Collection;
+import java.util.Date;
import java.util.List;
import ch.systemsx.cisd.authentication.ISessionManager;
@@ -305,11 +306,12 @@ public class ETLServiceLogger extends AbstractServerLogger implements IETLServic
}
public List<DeletedDataSet> listDeletedDataSets(String sessionToken,
- Long lastSeenDeletionEventIdOrNull)
+ Long lastSeenDeletionEventIdOrNull, Date maxDeletionDateOrNull)
{
logAccess(sessionToken, "listDeletedDataSets", "LAST_SEEN_EVENT(%s)",
(lastSeenDeletionEventIdOrNull == null ? "all" : "id > "
- + lastSeenDeletionEventIdOrNull));
+ + lastSeenDeletionEventIdOrNull), (maxDeletionDateOrNull == null ? "all"
+ : "maxDeletionDate > " + maxDeletionDateOrNull));
return null;
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
index e35b9f9ed2c..b98cf09840a 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/ExternalDataTable.java
@@ -276,7 +276,7 @@ public final class ExternalDataTable extends AbstractExternalDataBusinessObject
private static String getDeletionDescription(ExternalDataPE dataSet)
{
- return dataSet.getIdentifier();
+ return dataSet.getLocation();
}
public String uploadLoadedDataSetsToCIFEX(DataSetUploadContext uploadContext)
@@ -672,6 +672,8 @@ public final class ExternalDataTable extends AbstractExternalDataBusinessObject
archivingAction.execute(sessionToken, service, descriptions, userEmailOrNull);
} catch (Exception e)
{
+ // TODO KE: 2011-03-11 clear the archiving status (ARCHIVING_PENDING --> AVAILABLE,
+ // UNARCHIVING_PEDNING --> ARCHIVED
throw UserFailureException
.fromTemplate(
"Operation couldn't be performed for following datasets: %s. "
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IEventDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IEventDAO.java
index 99b41482fb5..8b46374ab61 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IEventDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/IEventDAO.java
@@ -16,12 +16,13 @@
package ch.systemsx.cisd.openbis.generic.server.dataaccess;
+import java.util.Date;
import java.util.List;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.EventPE;
-import ch.systemsx.cisd.openbis.generic.shared.dto.EventType;
import ch.systemsx.cisd.openbis.generic.shared.dto.EventPE.EntityType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.EventType;
/**
* <i>Data Access Object</i> for {@link EventPE}.
@@ -41,6 +42,7 @@ public interface IEventDAO extends IGenericDAO<EventPE>
/**
* Lists deleted data sets with the deletion event id greater than the specified one.
*/
- public List<DeletedDataSet> listDeletedDataSets(Long lastSeenDeletionEventIdOrNull);
+ public List<DeletedDataSet> listDeletedDataSets(Long lastSeenDeletionEventIdOrNull,
+ Date maxDeletionDataOrNull);
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAO.java
index c07440bdbab..d9dc931d888 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAO.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.generic.server.dataaccess.db;
import java.util.ArrayList;
+import java.util.Date;
import java.util.List;
import org.apache.log4j.Logger;
@@ -33,8 +34,8 @@ import ch.systemsx.cisd.openbis.generic.server.dataaccess.IEventDAO;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatabaseInstancePE;
import ch.systemsx.cisd.openbis.generic.shared.dto.EventPE;
-import ch.systemsx.cisd.openbis.generic.shared.dto.EventType;
import ch.systemsx.cisd.openbis.generic.shared.dto.EventPE.EntityType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.EventType;
/**
* Data access object for {@link EventPE}.
@@ -76,13 +77,18 @@ public class EventDAO extends AbstractGenericEntityDAO<EventPE> implements IEven
return result;
}
- public List<DeletedDataSet> listDeletedDataSets(Long lastSeenDeletionEventIdOrNull)
+ public List<DeletedDataSet> listDeletedDataSets(Long lastSeenDeletionEventIdOrNull,
+ Date maxDeletionDataOrNull)
{
final DetachedCriteria criteria = DetachedCriteria.forClass(EventPE.class);
if (lastSeenDeletionEventIdOrNull != null)
{
criteria.add(Restrictions.gt("id", lastSeenDeletionEventIdOrNull));
}
+ if (maxDeletionDataOrNull != null)
+ {
+ criteria.add(Restrictions.lt("registrationDate", maxDeletionDataOrNull));
+ }
criteria.add(Restrictions.eq("eventType", EventType.DELETION));
criteria.add(Restrictions.eq("entityType", EntityType.DATASET));
final List<EventPE> list = cast(getHibernateTemplate().findByCriteria(criteria));
@@ -98,7 +104,8 @@ public class EventDAO extends AbstractGenericEntityDAO<EventPE> implements IEven
ArrayList<DeletedDataSet> result = new ArrayList<DeletedDataSet>();
for (EventPE event : list)
{
- result.add(new DeletedDataSet(event.getIdentifier(), event.getId()));
+ result.add(new DeletedDataSet(event.getIdentifier(), event.getDescription(), event
+ .getId()));
}
return result;
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java
index 5049423d969..fb44cf7d7a0 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.generic.shared;
import java.util.Collection;
+import java.util.Date;
import java.util.List;
import org.springframework.transaction.annotation.Transactional;
@@ -429,7 +430,7 @@ public interface IETLLIMSService extends IServer, ISessionProvider
@Transactional(readOnly = true)
@RolesAllowed(RoleWithHierarchy.SPACE_ETL_SERVER)
public List<DeletedDataSet> listDeletedDataSets(String sessionToken,
- Long lastSeenDeletionEventIdOrNull);
+ Long lastSeenDeletionEventIdOrNull, Date maxDeletionDataOrNull);
/**
* List 'AVAILABLE' data sets (not locked) that match given criteria.
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/DeletedDataSet.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/DeletedDataSet.java
index 495728aa2a7..2da069c5140 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/DeletedDataSet.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/DeletedDataSet.java
@@ -17,6 +17,8 @@
package ch.systemsx.cisd.openbis.generic.shared.basic.dto;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
/**
* Describes a deleted data set.
@@ -25,18 +27,38 @@ import java.io.Serializable;
*/
public class DeletedDataSet implements Serializable
{
- private String identifier;
- private long eventId;
+ private static final long serialVersionUID = ServiceVersionHolder.VERSION;
+
+ private final String identifier;
+
+ private final long eventId;
+
+ private final String location; // the location where the data set existed before deletion
- public DeletedDataSet(String identifier, long eventId)
+ public static List<String> extractDataSetCodes(List<DeletedDataSet> dataSets)
+ {
+ List<String> result = new ArrayList<String>();
+ if (dataSets != null)
+ {
+ for (DeletedDataSet description : dataSets)
+ {
+ result.add(description.getIdentifier());
+ }
+ }
+ return result;
+ }
+
+ public DeletedDataSet(String identifier, String location, long eventId)
{
this.eventId = eventId;
this.identifier = identifier;
+ this.location = location;
}
- public DeletedDataSet()
+ public String getLocation()
{
+ return location;
}
public String getIdentifier()
@@ -44,21 +66,14 @@ public class DeletedDataSet implements Serializable
return identifier;
}
- public void setIdentifier(String identifier)
- {
- this.identifier = identifier;
- }
-
public long getEventId()
{
return eventId;
}
- public void setEventId(long eventId)
+ @Override
+ public String toString()
{
- this.eventId = eventId;
+ return "DeletedDataSet [identifier=" + identifier + "]";
}
-
- private static final long serialVersionUID = ServiceVersionHolder.VERSION;
-
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/DatasetDescription.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/DatasetDescription.java
index 5a7c2d94bbf..87f1f15ba8c 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/DatasetDescription.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/DatasetDescription.java
@@ -17,6 +17,8 @@
package ch.systemsx.cisd.openbis.generic.shared.dto;
import java.io.Serializable;
+import java.util.ArrayList;
+import java.util.List;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ServiceVersionHolder;
@@ -59,6 +61,19 @@ public class DatasetDescription implements Serializable
private String mainDataSetPath;
+ public static List<String> extractCodes(List<DatasetDescription> dataSets)
+ {
+ List<String> result = new ArrayList<String>();
+ if (dataSets != null)
+ {
+ for (DatasetDescription description : dataSets)
+ {
+ result.add(description.getDatasetCode());
+ }
+ }
+ return result;
+ }
+
public void setDatasetTypeCode(String datasetTypeCode)
{
this.datasetTypeCode = datasetTypeCode;
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAOTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAOTest.java
index c08d634c117..9334104d46b 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAOTest.java
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAOTest.java
@@ -23,9 +23,9 @@ import org.testng.AssertJUnit;
import org.testng.annotations.Test;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
+import ch.systemsx.cisd.openbis.generic.shared.dto.EventPE.EntityType;
import ch.systemsx.cisd.openbis.generic.shared.dto.EventType;
import ch.systemsx.cisd.openbis.generic.shared.dto.PersonPE;
-import ch.systemsx.cisd.openbis.generic.shared.dto.EventPE.EntityType;
import ch.systemsx.cisd.openbis.generic.shared.util.HibernateUtils;
/**
@@ -59,9 +59,16 @@ public class EventDAOTest extends AbstractDAOTest
}
}
+ private List<DeletedDataSet> listDataDeletionEvents(Long lastSeenDeletionEventIdOrNull,
+ Date maxDeletionDateOrNull)
+ {
+ return daoFactory.getEventDAO().listDeletedDataSets(lastSeenDeletionEventIdOrNull,
+ maxDeletionDateOrNull);
+ }
+
private List<DeletedDataSet> listDataDeletionEvents(Long lastSeenDeletionEventIdOrNull)
{
- return daoFactory.getEventDAO().listDeletedDataSets(lastSeenDeletionEventIdOrNull);
+ return listDataDeletionEvents(lastSeenDeletionEventIdOrNull, null);
}
@Test
@@ -148,8 +155,22 @@ public class EventDAOTest extends AbstractDAOTest
assertCorrectResult(0, result);
}
+ @Test
+ public void testListDeletedDataSetsWithSinceDate() throws Exception
+ {
+ Date beforeDate = new Date(0);
+ Date afterDate = new Date();
+ Date queryDate = new Date(afterDate.getTime() / 2);
+
+ saveEvent(EventType.DELETION, EntityType.DATASET, DELETE_ME + 1, AFTER + 1, beforeDate);
+ saveEvent(EventType.DELETION, EntityType.DATASET, DELETE_ME + 2, AFTER + 2, beforeDate);
+ saveEvent(EventType.DELETION, EntityType.DATASET, DELETE_ME + 3, AFTER + 3, afterDate);
+ List<DeletedDataSet> result = listDataDeletionEvents(SINCE, queryDate);
+ assertCorrectResult(2, result);
+ }
+
private void saveEvent(EventType eventType, EntityType entityType, String identifier,
- long eventId)
+ long eventId, Date date)
{
String description = eventType.name() + " " + entityType.name();
PersonPE person = getSystemPerson();
@@ -158,8 +179,14 @@ public class EventDAOTest extends AbstractDAOTest
.update(
"insert into events "
+ "(id, event_type, description, reason, pers_id_registerer, registration_timestamp, identifier, entity_type) "
- + "values(?, ?, ?, ?, ?, ?, ?, ?)", eventId, eventType.name(),
- description, description, personId, new Date(), identifier, entityType
+ + "values(?, ?, ?, ?, ?, ?, ?, ?)", eventId, eventType.name(), description,
+ description, personId, date, identifier, entityType
.name());
}
+
+ private void saveEvent(EventType eventType, EntityType entityType, String identifier,
+ long eventId)
+ {
+ saveEvent(eventType, entityType, identifier, eventId, new Date());
+ }
}
\ No newline at end of file
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java.expected b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java.expected
index 5049423d969..fb44cf7d7a0 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java.expected
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/IETLLIMSService.java.expected
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.generic.shared;
import java.util.Collection;
+import java.util.Date;
import java.util.List;
import org.springframework.transaction.annotation.Transactional;
@@ -429,7 +430,7 @@ public interface IETLLIMSService extends IServer, ISessionProvider
@Transactional(readOnly = true)
@RolesAllowed(RoleWithHierarchy.SPACE_ETL_SERVER)
public List<DeletedDataSet> listDeletedDataSets(String sessionToken,
- Long lastSeenDeletionEventIdOrNull);
+ Long lastSeenDeletionEventIdOrNull, Date maxDeletionDataOrNull);
/**
* List 'AVAILABLE' data sets (not locked) that match given criteria.
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java
index 2d14a42b417..843b5f9b07e 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MLArchiverTask.java
@@ -33,6 +33,7 @@ import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ArchiverTaskCon
import ch.systemsx.cisd.openbis.dss.generic.shared.DataSourceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
@@ -247,9 +248,13 @@ public class MLArchiverTask extends AbstractArchiverProcessingPlugin
}
@Override
- protected DatasetProcessingStatuses doDeleteFromArchive(List<DatasetDescription> datasets,
- ArchiverTaskContext context)
+ protected DatasetProcessingStatuses doDeleteFromArchive(List<DeletedDataSet> datasets)
{
- return createStatuses(Status.OK, datasets, Operation.DELETE_FROM_ARCHIVE);
+ DatasetProcessingStatuses statuses = new DatasetProcessingStatuses();
+ for (DeletedDataSet dataset : datasets)
+ {
+ statuses.addResult(dataset.getIdentifier(), Status.OK, Operation.DELETE_FROM_ARCHIVE);
+ }
+ return statuses;
}
}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MetabolDatabaseUpdater.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MetabolDatabaseUpdater.java
index 81c37b7a4f9..90e7558bf75 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MetabolDatabaseUpdater.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MetabolDatabaseUpdater.java
@@ -15,13 +15,13 @@
*/
package ch.systemsx.cisd.yeastx.etl;
-import ch.systemsx.cisd.etlserver.plugins.DataSetDeletionMaintenanceTask;
+import ch.systemsx.cisd.etlserver.plugins.DeleteFromExternalDBMaintenanceTask;
/**
* Maintenance task deleting from metabol database data sets which have been deleted from openbis.
*
* @author Izabela Adamczyk
*/
-public class MetabolDatabaseUpdater extends DataSetDeletionMaintenanceTask
+public class MetabolDatabaseUpdater extends DeleteFromExternalDBMaintenanceTask
{
}
\ No newline at end of file
--
GitLab