diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java
index c4ab2f6c206d7283d291e530d2a292108f132cb0..1e1790e8f0077ff6377d2958c8af70fb17d7324b 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java
@@ -24,6 +24,7 @@ import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.mail.IMailClient;
@@ -130,7 +131,7 @@ final class DataStrategyStore implements IDataStrategyStore
return dataStoreStrategies.get(DataStoreStrategyKey.UNIDENTIFIED);
}
final SampleIdentifier sampleIdentifier = dataSetInfo.getSampleIdentifier();
- final SamplePE sample = limsService.tryGetSampleWithExperiment(sampleIdentifier);
+ final SamplePE sample = tryGetSample(sampleIdentifier);
final ExperimentPE experiment = (sample == null) ? null : sample.getExperiment();
if (experiment == null)
{
@@ -181,6 +182,17 @@ final class DataStrategyStore implements IDataStrategyStore
return dataStoreStrategies.get(DataStoreStrategyKey.IDENTIFIED);
}
+ private SamplePE tryGetSample(final SampleIdentifier sampleIdentifier)
+ {
+ try
+ {
+ return limsService.tryGetSampleWithExperiment(sampleIdentifier);
+ } catch (UserFailureException ex)
+ {
+ return null;
+ }
+ }
+
private void sendEmail(final String message, final ExperimentIdentifier experimentIdentifier,
final String recipientMail)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java
index 7b05e940bad1d3e377810c9f16a37444a4af1d37..ad063278c3f17bf0a86be7337e11eec34cdecf2e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java
@@ -328,7 +328,8 @@ public class DefaultDataSetInfoExtractor extends AbstractDataSetInfoExtractor
}
}
- private List<NewProperty> extractDataSetProperties(File incomingDataSetPath, String fileName)
+ private static List<NewProperty> extractDataSetProperties(File incomingDataSetPath,
+ String fileName)
{
List<NewProperty> result = new ArrayList<NewProperty>();
if (fileName != null && incomingDataSetPath.isDirectory())
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
index a7e3ed2bca1f27ecdfe75db3279707a7cb81d396..c54e951f5fb2129bc0a71a31d3637bb32ef22a86 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
@@ -356,7 +356,8 @@ public final class ETLDaemon
final TransferredDataSetHandler pathHandler =
new TransferredDataSetHandler(dssCode, plugin, authorizedLimsService,
mailProperties, highwaterMarkWatcher, notifySuccessfulRegistration,
- threadParameters.useIsFinishedMarkerFile());
+ threadParameters.useIsFinishedMarkerFile(), threadParameters
+ .deleteUnidentified());
pathHandler.setProcessorFactories(processorFactories);
final HighwaterMarkDirectoryScanningHandler directoryScanningHandler =
createDirectoryScanningHandler(pathHandler, highwaterMarkWatcher,
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
index 8475ddd53644e2ee04738b2bfe83c162965099f6..9e3978a5227e3b10645fc2cf7534f1eaf5d2820e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java
@@ -56,6 +56,8 @@ public final class ThreadParameters
private static final String INCOMING_DIR = "incoming-dir";
+ private static final String DELETE_UNIDENTIFIED_KEY = "delete-unidentified";
+
/**
* The (local) directory to monitor for new files and directories to move to the remote side.
* The directory where data to be processed by the ETL server become available.
@@ -70,6 +72,8 @@ public final class ThreadParameters
private final boolean useIsFinishedMarkerFile;
+ private final boolean deleteUnidentified;
+
/**
* @param threadProperties parameters for one processing thread together with general
* parameters.
@@ -83,6 +87,8 @@ public final class ThreadParameters
PropertyUtils.getProperty(threadProperties, INCOMING_DATA_COMPLETENESS_CONDITION,
INCOMING_DATA_COMPLETENESS_CONDITION_MARKER_FILE);
this.useIsFinishedMarkerFile = parseCompletenessCondition(completenessCondition);
+ this.deleteUnidentified =
+ "true".equals(threadProperties.getProperty(DELETE_UNIDENTIFIED_KEY, "false"));
this.threadName = threadName;
}
@@ -185,6 +191,7 @@ public final class ThreadParameters
useIsFinishedMarkerFile ? "marker file exists"
: "no write access for some period";
logLine("Condition of incoming data completeness: %s.", completenessCond);
+ logLine("Delete unidentified: '%s'.", deleteUnidentified);
}
}
@@ -200,4 +207,9 @@ public final class ThreadParameters
{
return threadName;
}
+
+ public boolean deleteUnidentified()
+ {
+ return deleteUnidentified;
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
index 3fe2c43510195fa1e3bd43bd9891135eb5e778dd..6bd6aac1304a4ede944fb5c713f39d08c9e136a5 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
@@ -127,6 +127,8 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
private boolean stopped = false;
+ private boolean deleteUnidentified = false;
+
private Map<String, IProcessorFactory> processorFactories =
Collections.<String, IProcessorFactory> emptyMap();
@@ -142,18 +144,20 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
public TransferredDataSetHandler(String dssCode, final IETLServerPlugin plugin,
final IEncapsulatedOpenBISService limsService, final Properties mailProperties,
final HighwaterMarkWatcher highwaterMarkWatcher,
- final boolean notifySuccessfulRegistration, boolean useIsFinishedMarkerFile)
+ final boolean notifySuccessfulRegistration, boolean useIsFinishedMarkerFile,
+ boolean deleteUnidentified)
{
this(dssCode, plugin.getStorageProcessor(), plugin, limsService, new MailClient(
- mailProperties), notifySuccessfulRegistration, useIsFinishedMarkerFile);
+ mailProperties), notifySuccessfulRegistration, useIsFinishedMarkerFile,
+ deleteUnidentified);
}
TransferredDataSetHandler(String dssCode,
final IStoreRootDirectoryHolder storeRootDirectoryHolder,
final IETLServerPlugin plugin, final IEncapsulatedOpenBISService limsService,
final IMailClient mailClient, final boolean notifySuccessfulRegistration,
- boolean useIsFinishedMarkerFile)
+ boolean useIsFinishedMarkerFile, boolean deleteUnidentified)
{
assert dssCode != null : "Unspecified data store code";
@@ -175,6 +179,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
this.registrationLock = new ReentrantLock();
this.fileOperations = FileOperations.getMonitoredInstanceForCurrentThread();
this.useIsFinishedMarkerFile = useIsFinishedMarkerFile;
+ this.deleteUnidentified = deleteUnidentified;
}
public final void setProcessorFactories(final Map<String, IProcessorFactory> processorFactories)
@@ -497,6 +502,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
}
final StopWatch watch = new StopWatch();
watch.start();
+ ExternalData data = createExternalData();
File dataFile =
storageProcessor.storeData(sample, dataSetInformation, typeExtractor,
mailClient, incomingDataSetFile, baseDirectoryHolder
@@ -518,7 +524,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
try
{
errorMessageTemplate = DATA_SET_REGISTRATION_FAILURE_TEMPLATE;
- plainRegisterDataSet(relativePath, availableFormat, isCompleteFlag);
+ plainRegisterDataSet(data, relativePath, availableFormat, isCompleteFlag);
clean();
clean();
if (processorOrNull == null)
@@ -612,19 +618,19 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
final void moveDataSet()
{
final boolean ok =
- FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder
- .getTargetFile());
+ deleteUnidentified ? (FileUtilities.deleteRecursively(incomingDataSetFile))
+ : FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder
+ .getTargetFile());
if (ok)
{
clean();
}
}
- private final void plainRegisterDataSet(final String relativePath,
+ private final void plainRegisterDataSet(ExternalData data, final String relativePath,
final StorageFormat storageFormat, final BooleanOrUnknown isCompleteFlag)
{
- final ExternalData data =
- createExternalData(relativePath, storageFormat, isCompleteFlag);
+ updateExternalData(data, relativePath, storageFormat, isCompleteFlag);
// Finally: register the data set in the database.
limsService.registerDataSet(dataSetInformation, data);
}
@@ -763,18 +769,23 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
return null;
}
- private final ExternalData createExternalData(final String relativePath,
+ private final ExternalData updateExternalData(ExternalData data, final String relativePath,
final StorageFormat storageFormat, final BooleanOrUnknown isCompleteFlag)
+ {
+ data.setComplete(isCompleteFlag);
+ data.setLocation(relativePath);
+ data.setStorageFormat(storageFormat);
+ return data;
+ }
+
+ private ExternalData createExternalData()
{
final ExtractableData extractableData = dataSetInformation.getExtractableData();
final ExternalData data = BeanUtils.createBean(ExternalData.class, extractableData);
- data.setLocation(relativePath);
data.setLocatorType(typeExtractor.getLocatorType(incomingDataSetFile));
data.setDataSetType(typeExtractor.getDataSetType(incomingDataSetFile));
data.setFileFormatType(typeExtractor.getFileFormatType(incomingDataSetFile));
data.setMeasured(typeExtractor.isMeasuredData(incomingDataSetFile));
- data.setStorageFormat(storageFormat);
- data.setComplete(isCompleteFlag);
data.setDataStoreCode(dssCode);
return data;
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexDataSetInfoExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexDataSetInfoExtractor.java
new file mode 100644
index 0000000000000000000000000000000000000000..afdc63b01c11b964761286a00ec85a890f4d9472
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexDataSetInfoExtractor.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.cifex;
+
+import java.io.File;
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.etlserver.IDataSetInfoExtractor;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.openbis.generic.shared.basic.DataSetUploadInfo;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFactory;
+
+/**
+ * {@link IDataSetInfoExtractor} extracting group and sample from CIFEX comment.
+ *
+ * @author Izabela Adamczyk
+ */
+public class CifexDataSetInfoExtractor implements IDataSetInfoExtractor
+{
+
+ public CifexDataSetInfoExtractor(final Properties globalProperties)
+ {
+ }
+
+ public DataSetInformation getDataSetInformation(File incomingDataSetPath,
+ IEncapsulatedOpenBISService openbisService) throws UserFailureException,
+ EnvironmentFailureException
+ {
+ assert incomingDataSetPath != null : "Incoming data set path can not be null.";
+
+ DataSetUploadInfo info = CifexExtratorHelper.getDataSetUploadInfo(incomingDataSetPath);
+ final DataSetInformation dataSetInformation = new DataSetInformation();
+ SampleIdentifier identifier = SampleIdentifierFactory.parse(info.getSample());
+ dataSetInformation.setSampleCode(identifier.getSampleCode());
+ if (identifier.isGroupLevel())
+ {
+ dataSetInformation.setGroupCode(identifier.getGroupLevel().getGroupCode());
+ }
+ dataSetInformation.setUploadingUserEmail(CifexExtratorHelper
+ .getUploadingUserEmail(incomingDataSetPath));
+ return dataSetInformation;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexExtratorHelper.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexExtratorHelper.java
new file mode 100644
index 0000000000000000000000000000000000000000..201fa22e6426f9044d030b5e87615defa191d4e5
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexExtratorHelper.java
@@ -0,0 +1,64 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.cifex;
+
+import java.io.File;
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.generic.shared.basic.DataSetUploadInfo;
+import ch.systemsx.cisd.openbis.generic.shared.basic.DataSetUploadInfo.DataSetUploadInfoHelper;
+
+/**
+ * @author Izabela Adamczyk
+ */
+public class CifexExtratorHelper
+{
+ private static final String COMMENT_KEY = "comment";
+
+ private static final String REQUEST_PROPERTIES_FILE = "request.properties";
+
+ private static final String UPLOADING_USER_EMAIL_KEY = "user-email";
+
+ public static DataSetUploadInfo getDataSetUploadInfo(File incomingDataSetPath)
+ {
+ Properties properties = loadProperties(incomingDataSetPath, REQUEST_PROPERTIES_FILE);
+ String comment = properties.getProperty(COMMENT_KEY);
+ DataSetUploadInfo info = DataSetUploadInfoHelper.extractFromCifexComment(comment);
+ return info;
+ }
+
+ public static String getUploadingUserEmail(File incomingDataSetPath)
+ {
+ Properties properties = loadProperties(incomingDataSetPath, REQUEST_PROPERTIES_FILE);
+ return properties.getProperty(UPLOADING_USER_EMAIL_KEY);
+ }
+
+ private static Properties loadProperties(File incomingDataSetPath, String fileName)
+ {
+ File propertiesFile = new File(incomingDataSetPath, fileName);
+ if (propertiesFile.isFile())
+ {
+ return PropertyUtils.loadProperties(propertiesFile.getPath());
+ } else
+ {
+ throw new UserFailureException("Request properties file '" + propertiesFile
+ + "' does not exist or is not a 'normal' file.");
+ }
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexTypeExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexTypeExtractor.java
new file mode 100644
index 0000000000000000000000000000000000000000..0e72f721113b3992aac9894f87a6e7229082cb9e
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexTypeExtractor.java
@@ -0,0 +1,73 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.cifex;
+
+import java.io.File;
+import java.util.Properties;
+
+import ch.systemsx.cisd.etlserver.ITypeExtractor;
+import ch.systemsx.cisd.etlserver.SimpleTypeExtractor;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.FileFormatType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.LocatorType;
+
+/**
+ * {@link ITypeExtractor} which extracts data set type and file format from the CIFEX comment saved
+ * in 'request.properties' file. <br>
+ * Locator type, processor type and 'is measured' values are calculated by corresponding methods of
+ * {@link SimpleTypeExtractor}.
+ *
+ * @author Izabela Adamczyk
+ */
+public class CifexTypeExtractor implements ITypeExtractor
+{
+
+ private ITypeExtractor simpleTypeExtractor;
+
+ public CifexTypeExtractor(final Properties properties)
+ {
+ simpleTypeExtractor = new SimpleTypeExtractor(properties);
+ }
+
+ public DataSetType getDataSetType(File incomingDataSetPath)
+ {
+ return new DataSetType(CifexExtratorHelper.getDataSetUploadInfo(incomingDataSetPath)
+ .getDataSetType());
+ }
+
+ public FileFormatType getFileFormatType(File incomingDataSetPath)
+ {
+ return new FileFormatType(CifexExtratorHelper.getDataSetUploadInfo(incomingDataSetPath)
+ .getFileType());
+ }
+
+ public LocatorType getLocatorType(File incomingDataSetPath)
+ {
+ return simpleTypeExtractor.getLocatorType(incomingDataSetPath);
+ }
+
+ public String getProcessorType(File incomingDataSetPath)
+ {
+ return simpleTypeExtractor.getProcessorType(incomingDataSetPath);
+ }
+
+ public boolean isMeasuredData(File incomingDataSetPath)
+ {
+ return simpleTypeExtractor.isMeasuredData(incomingDataSetPath);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java
index ea2218e4c86cbc0de359ed72a16874ca07bc02c2..643c3c26a0a1799105b970daf2764cc36cdd8de3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java
@@ -82,11 +82,26 @@ public class DataSetInformation implements Serializable
*/
private ExtractableData extractableData = new ExtractableData();
+ /**
+ * Email of uploading user.
+ */
+ private String uploadingUserEmailOrNull;
+
/** This constructor is for serialization. */
public DataSetInformation()
{
}
+ public String tryGetUploadingUserEmail()
+ {
+ return uploadingUserEmailOrNull;
+ }
+
+ public void setUploadingUserEmail(String uploadingUserEmail)
+ {
+ this.uploadingUserEmailOrNull = uploadingUserEmail;
+ }
+
public final BooleanOrUnknown getIsCompleteFlag()
{
return isCompleteFlag;
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
index ac6c6b57364c801383feb6e00fac94424c8056ec..4c6681fb6655be62cffde5b0dab70c25af7cb0f1 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
@@ -277,7 +277,7 @@ public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestC
authorizedLimsService.setPassword("p");
handler =
new TransferredDataSetHandler("dss", storageProcessor, plugin,
- authorizedLimsService, mailClient, true, true);
+ authorizedLimsService, mailClient, true, true, false);
handler.setProcessorFactories(map);
dataSetInformation = new DataSetInformation();
@@ -842,6 +842,11 @@ public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestC
will(throwException(new Exception("Could store data by storage processor")));
one(storageProcessor).unstoreData(with(equal(folder)), with(equal(baseDir)));
+
+ allowing(typeExtractor).getLocatorType(folder);
+ allowing(typeExtractor).getDataSetType(folder);
+ allowing(typeExtractor).getFileFormatType(folder);
+ allowing(typeExtractor).isMeasuredData(folder);
}
});
final LogMonitoringAppender appender =