diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ProcessDatasetsCommand.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ProcessDatasetsCommand.java
index 0e96916748266ceb1c0595952c6ecc372f6e20d1..d77e41bfcaf4c3230ddcf4cb5bb9508155f21ecd 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ProcessDatasetsCommand.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ProcessDatasetsCommand.java
@@ -21,11 +21,13 @@ import java.util.List;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.mail.IMailClient;
import ch.systemsx.cisd.common.mail.MailClient;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IProcessingPluginTask;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ProcessingStatus;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DatastoreServiceDescription;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
@@ -66,9 +68,10 @@ public class ProcessDatasetsCommand implements IDataSetCommand
public void execute(File store)
{
String errorMessageOrNull = null;
+ ProcessingStatus processingStatusOrNull = null;
try
{
- task.process(datasets);
+ processingStatusOrNull = task.process(datasets);
} catch (RuntimeException e)
{
// exception message should be readable for users
@@ -91,26 +94,33 @@ public class ProcessDatasetsCommand implements IDataSetCommand
}
return;
}
- createContentAndSendMessage(errorMessageOrNull);
+ createContentAndSendMessage(errorMessageOrNull, processingStatusOrNull);
}
}
- private void createContentAndSendMessage(String errorMessageOrNull)
+ private void createContentAndSendMessage(String errorMessageOrNull,
+ ProcessingStatus processingStatusOrNull)
{
final StringBuilder contentBuilder = new StringBuilder();
final String subject;
if (errorMessageOrNull != null)
{
// create error message content
- subject = getShortDescription("Failed to perform ");
+ subject = getShortDescription(" processing failed");
contentBuilder.append(getDescription(subject));
contentBuilder.append("\n\nError message:\n");
contentBuilder.append(errorMessageOrNull);
} else
{
// create success message content
- subject = getShortDescription("Finished ");
- contentBuilder.append(getDescription(subject));
+ subject = getShortDescription(" processing finished");
+ if (processingStatusOrNull != null)
+ {
+ contentBuilder.append(generateDescription(processingStatusOrNull));
+ } else
+ {
+ contentBuilder.append(getDescription(subject));
+ }
}
sendMessage(subject, contentBuilder.toString(), userEmailOrNull);
}
@@ -126,14 +136,14 @@ public class ProcessDatasetsCommand implements IDataSetCommand
mailClient.sendMessage(subject, content, null, null, recipient);
}
- private String getShortDescription(String prefix)
+ private String getShortDescription(String suffix)
{
- return String.format("%s'%s'", prefix, serviceDescription.getLabel());
+ return String.format("'%s'%s", serviceDescription.getLabel(), suffix);
}
private String getDescription(String prefix)
{
- return String.format("%s on data set(s): [%s]", prefix, getDataSetCodes());
+ return String.format("%s on data set(s): \n%s", prefix, getDataSetCodes(datasets));
}
public String getDescription()
@@ -141,7 +151,7 @@ public class ProcessDatasetsCommand implements IDataSetCommand
return getDescription(getShortDescription(""));
}
- public String getDataSetCodes()
+ private static String getDataSetCodes(List<DatasetDescription> datasets)
{
if (datasets.isEmpty())
{
@@ -158,4 +168,25 @@ public class ProcessDatasetsCommand implements IDataSetCommand
return sb.toString();
}
}
+
+ private static String generateDescription(ProcessingStatus processingStatus)
+ {
+ StringBuilder sb = new StringBuilder();
+ sb.append("Data sets:\n");
+ List<DatasetDescription> successfullyProcessed =
+ processingStatus.getDatasetsByStatus(Status.OK);
+ if (successfullyProcessed.isEmpty() == false)
+ {
+ sb.append("- successfully processed: ");
+ sb.append(getDataSetCodes(successfullyProcessed));
+ }
+ List<Status> errorStatuses = processingStatus.getErrorStatuses();
+ for (Status errorStatus : errorStatuses)
+ {
+ sb.append("\n- " + errorStatus.tryGetErrorMessage() + ": ");
+ sb.append(getDataSetCodes(successfullyProcessed));
+ }
+ return sb.toString();
+ }
+
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoProcessingPlugin.java
index b8b154f0563782ca6fdd918f8f87c512b7d14186..7a531679f3d403bdd93406dcd2217cf8e4d8e0ba 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/demo/DemoProcessingPlugin.java
@@ -21,6 +21,7 @@ import java.util.List;
import java.util.Properties;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IProcessingPluginTask;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ProcessingStatus;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
/**
@@ -36,7 +37,7 @@ public class DemoProcessingPlugin implements IProcessingPluginTask
{
}
- public void process(List<DatasetDescription> datasets)
+ public ProcessingStatus process(List<DatasetDescription> datasets)
{
System.out.println("Processing of the following datasets has been requested: " + datasets);
System.out.println("sleeping for 10 sec");
@@ -48,6 +49,6 @@ public class DemoProcessingPlugin implements IProcessingPluginTask
ex.printStackTrace();
}
System.out.println("Processing done.");
+ return null;
}
-
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractDropboxProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractDropboxProcessingPlugin.java
index 4cfeb8c433b940217430760211b5bd9ebb0b0625..7d252c834d97c4608174153096af0671233f8f2e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractDropboxProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractDropboxProcessingPlugin.java
@@ -22,10 +22,12 @@ import java.util.Properties;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.IProcessingPluginTask;
+import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ProcessingStatus;
import ch.systemsx.cisd.openbis.dss.generic.shared.IPostRegistrationDatasetHandler;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
@@ -42,7 +44,13 @@ abstract public class AbstractDropboxProcessingPlugin extends AbstractDatastoreP
private static final long serialVersionUID = 1L;
final static Logger operationLog =
- LogFactory.getLogger(LogCategory.OPERATION, IPostRegistrationDatasetHandler.class);
+ LogFactory.getLogger(LogCategory.OPERATION, AbstractDropboxProcessingPlugin.class);
+
+ private final static String MISSING_DIRECTORY_MSG = "with missing directory";
+
+ private final static String EMPTY_DIRECTORY_MSG = "with empty directory";
+
+ private final static String MORE_THAN_ONE_ITEM_MSG = "with more than one item in the directory";
private final IPostRegistrationDatasetHandler dropboxHandler;
@@ -57,30 +65,40 @@ abstract public class AbstractDropboxProcessingPlugin extends AbstractDatastoreP
this.dropboxHandler = dropboxHandler;
}
- public void process(List<DatasetDescription> datasets)
+ public ProcessingStatus process(List<DatasetDescription> datasets)
{
+ final ProcessingStatus result = new ProcessingStatus();
for (DatasetDescription dataset : datasets)
{
- File originalDir = getDataSubDir(dataset);
- if (originalDir.isDirectory() == false)
- {
- operationLog
- .warn("Dataset directory does not exist and will be silently excluded from the processing: "
- + originalDir.getPath());
- continue;
- }
- File[] datasetFiles = FileUtilities.listFiles(originalDir);
- if (datasetFiles.length == 1)
- {
- DataSetInformation datasetInfo = createDatasetInfo(dataset);
- dropboxHandler.handle(datasetFiles[0], datasetInfo);
- } else if (datasetFiles.length > 1)
- {
- operationLog.error(String.format(
- "Exactly one item was expected in the '%s' directory,"
- + " but %d have been found. Nothing will be processed.",
- originalDir.getParent(), datasetFiles.length));
- }
+ Status status = processDataset(dataset);
+ result.addDatasetStatus(dataset, status);
+ }
+ return result;
+ }
+
+ private Status processDataset(DatasetDescription dataset)
+ {
+ File originalDir = getDataSubDir(dataset);
+ if (originalDir.isDirectory() == false)
+ {
+ operationLog
+ .warn("Dataset directory does not exist and will be silently excluded from the processing: "
+ + originalDir.getPath());
+ return Status.createError(MISSING_DIRECTORY_MSG);
+ }
+ File[] datasetFiles = FileUtilities.listFiles(originalDir);
+ if (datasetFiles.length == 1)
+ {
+ DataSetInformation datasetInfo = createDatasetInfo(dataset);
+ return dropboxHandler.handle(datasetFiles[0], datasetInfo);
+ } else
+ {
+ operationLog.error(String.format("Exactly one item was expected in the '%s' directory,"
+ + " but %d have been found. Nothing will be processed.", originalDir
+ .getParent(), datasetFiles));
+ final String errorMsg =
+ datasetFiles.length > 1 ? MORE_THAN_ONE_ITEM_MSG : EMPTY_DIRECTORY_MSG;
+ return Status.createError(errorMsg);
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java
index 035c6c871bc700dbbe6c3411cc904f0a2d763f9a..cf3419a2e604991a719d56f3c98bcf080703da8d 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetCopier.java
@@ -23,7 +23,6 @@ import java.util.Properties;
import ch.rinn.restrictions.Private;
import ch.systemsx.cisd.base.utilities.OSUtilities;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
-import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.IPathCopier;
@@ -55,6 +54,10 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
@Private
static final String RSYNC_PASSWORD_FILE_KEY = "rsync-password-file";
+ private static final String ALREADY_EXIST_MSG = "already exist";
+
+ private static final String COPYING_FAILED_MSG = "copying failed";
+
private static final String RSYNC_EXEC = "rsync";
private static final String SSH_EXEC = "ssh";
@@ -128,18 +131,40 @@ public class DataSetCopier extends AbstractDropboxProcessingPlugin
return copier;
}
- public void handle(File originalData, DataSetInformation dataSetInformation)
+ public Status handle(File originalData, DataSetInformation dataSetInformation)
{
+ if (checkDestinationExists())
+ {
+ operationLog.error("Destination directory '" + destination.getPath()
+ + "' already exists - dataset files will not be copied.");
+ return Status.createError(ALREADY_EXIST_MSG);
+ }
Status status =
getCopier().copyToRemote(originalData, destination, host, rsyncModule,
rsyncPasswordFile);
if (status.isError())
{
- throw new EnvironmentFailureException("Could not copy data set "
- + dataSetInformation.getDataSetCode() + " to destination folder '"
- + destination + "'" + (host != null ? " on host '" + host + "'" : "")
+ operationLog.error("Could not copy data set " + dataSetInformation.getDataSetCode()
+ + " to destination folder '" + destination + "'"
+ + (host != null ? " on host '" + host + "'" : "")
+ (rsyncModule != null ? " for rsync module '" + rsyncModule + "'" : "")
+ ": " + status.tryGetErrorMessage());
+ return Status.createError(COPYING_FAILED_MSG);
+ }
+ return status;
+ }
+
+ private boolean checkDestinationExists()
+ {
+ if (host != null)
+ {
+ // TODO 2010-01-29, Piotr Buczek: check if directory already exists
+ // check remotely using ssh
+ return false;
+ } else
+ {
+ // check locally
+ return destination.exists();
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/IProcessingPluginTask.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/IProcessingPluginTask.java
index 6b9bd6a56ee06da65684b7090eb7574e51f6b313..6bcbcf8c376f463b41a288e6cdc6e1d8ae6fb0f4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/IProcessingPluginTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/IProcessingPluginTask.java
@@ -30,7 +30,11 @@ public interface IProcessingPluginTask extends Serializable
{
/**
* Processes asynchronously the specified datasets.
+ *
+ * @returns {@link ProcessingStatus} of the finished processing with statuses of processing for
+ * all scheduled data sets or null if processing succeeded for all datasets and no
+ * additional information is provided.
*/
- void process(List<DatasetDescription> datasets);
+ ProcessingStatus process(List<DatasetDescription> datasets);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/ProcessingStatus.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/ProcessingStatus.java
new file mode 100644
index 0000000000000000000000000000000000000000..805dc09bd9cfcf11a0798515df552941ab4ba980
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/ProcessingStatus.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks;
+
+import java.util.ArrayList;
+import java.util.LinkedHashMap;
+import java.util.List;
+import java.util.Map;
+
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+
+/**
+ * Holder of finished processing {@link Status}es of all handled data sets.
+ *
+ * @author Piotr Buczek
+ */
+public class ProcessingStatus
+{
+
+ private Map<Status, List<DatasetDescription>> datasetByStatus =
+ new LinkedHashMap<Status, List<DatasetDescription>>();
+
+ public void addDatasetStatus(DatasetDescription dataset, Status status)
+ {
+ List<DatasetDescription> datasets = datasetByStatus.get(status);
+ if (datasets == null)
+ {
+ datasets = new ArrayList<DatasetDescription>();
+ datasetByStatus.put(status, datasets);
+ }
+ datasets.add(dataset);
+ }
+
+ public List<Status> getErrorStatuses()
+ {
+ List<Status> result = new ArrayList<Status>(datasetByStatus.keySet());
+ result.remove(Status.OK);
+ return result;
+ }
+
+ public List<DatasetDescription> getDatasetsByStatus(Status status)
+ {
+ return datasetByStatus.get(status);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IPostRegistrationDatasetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IPostRegistrationDatasetHandler.java
index c91fbd3abb92348cbb0a769f9ffc7d358a735c35..7196facb9ac0602ecf06138180e1037597d8d194 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IPostRegistrationDatasetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IPostRegistrationDatasetHandler.java
@@ -18,20 +18,23 @@ package ch.systemsx.cisd.openbis.dss.generic.shared;
import java.io.File;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
* Handler of data sets after successful registration in openBIS.
- *
+ *
* @author Franz-Josef Elmer
*/
public interface IPostRegistrationDatasetHandler
{
/**
- * Handles specified original data file by using specified data set information.
- * Note, that <code>originalData</code> is already the path inside the data store.
+ * Handles specified original data file by using specified data set information. Note, that
+ * <code>originalData</code> is already the path inside the data store.
+ *
+ * @return {@link Status} of the operation.
*/
- public void handle(File originalData, final DataSetInformation dataSetInformation);
+ public Status handle(File originalData, final DataSetInformation dataSetInformation);
/**
* Reverts the previous invocation of {@link #handle(File, DataSetInformation)}.
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/AbstractDatasetDropboxHandler.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/AbstractDatasetDropboxHandler.java
index a47276ea2ceba836b63453087c3fd5a6a9b0fabb..5919258de1471db349f8f59cf8fc39a5946fc6a6 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/AbstractDatasetDropboxHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/AbstractDatasetDropboxHandler.java
@@ -26,6 +26,7 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.FileOperations;
import ch.systemsx.cisd.common.filesystem.IFileOperations;
import ch.systemsx.cisd.common.logging.LogCategory;
@@ -44,7 +45,8 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
*
* @author Tomasz Pylak
*/
-abstract public class AbstractDatasetDropboxHandler implements Serializable, IPostRegistrationDatasetHandler
+abstract public class AbstractDatasetDropboxHandler implements Serializable,
+ IPostRegistrationDatasetHandler
{
private static final long serialVersionUID = 1L;
@@ -123,7 +125,7 @@ abstract public class AbstractDatasetDropboxHandler implements Serializable, IPo
return file;
}
- public final void handle(File originalData, final DataSetInformation dataSetInformation)
+ public final Status handle(File originalData, final DataSetInformation dataSetInformation)
{
File dropboxDir = tryGetDropboxDir(originalData, dataSetInformation);
if (dropboxDir != null)
@@ -132,6 +134,7 @@ abstract public class AbstractDatasetDropboxHandler implements Serializable, IPo
createDropboxDestinationFileName(dataSetInformation, originalData);
copy(originalData, dropboxDir, destinationFileName);
}
+ return Status.OK;
}
private void copy(File originalData, File dropboxDir, String destinationFileName)