From c90c72acc3325a3186fb9c2ad017138fb7b6ded5 Mon Sep 17 00:00:00 2001 From: izabel <izabel> Date: Mon, 22 Jun 2009 21:15:19 +0000 Subject: [PATCH] [LMS-950] DDS: type extractor and data info extractor compatible with cifex trigger SVN: 11486 --- .../cisd/etlserver/DataStrategyStore.java | 14 +++- .../DefaultDataSetInfoExtractor.java | 3 +- .../ch/systemsx/cisd/etlserver/ETLDaemon.java | 3 +- .../cisd/etlserver/ThreadParameters.java | 12 +++ .../etlserver/TransferredDataSetHandler.java | 37 ++++++---- .../cifex/CifexDataSetInfoExtractor.java | 62 ++++++++++++++++ .../etlserver/cifex/CifexExtratorHelper.java | 64 ++++++++++++++++ .../etlserver/cifex/CifexTypeExtractor.java | 73 +++++++++++++++++++ .../shared/dto/DataSetInformation.java | 15 ++++ .../TransferredDataSetHandlerTest.java | 7 +- 10 files changed, 273 insertions(+), 17 deletions(-) create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexDataSetInfoExtractor.java create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexExtratorHelper.java create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexTypeExtractor.java diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java index c4ab2f6c206..1e1790e8f00 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DataStrategyStore.java @@ -24,6 +24,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.log4j.Logger; import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; +import ch.systemsx.cisd.common.exceptions.UserFailureException; import ch.systemsx.cisd.common.logging.LogCategory; import ch.systemsx.cisd.common.logging.LogFactory; import ch.systemsx.cisd.common.mail.IMailClient; @@ -130,7 +131,7 @@ final class DataStrategyStore implements IDataStrategyStore return dataStoreStrategies.get(DataStoreStrategyKey.UNIDENTIFIED); } final SampleIdentifier sampleIdentifier = dataSetInfo.getSampleIdentifier(); - final SamplePE sample = limsService.tryGetSampleWithExperiment(sampleIdentifier); + final SamplePE sample = tryGetSample(sampleIdentifier); final ExperimentPE experiment = (sample == null) ? null : sample.getExperiment(); if (experiment == null) { @@ -181,6 +182,17 @@ final class DataStrategyStore implements IDataStrategyStore return dataStoreStrategies.get(DataStoreStrategyKey.IDENTIFIED); } + private SamplePE tryGetSample(final SampleIdentifier sampleIdentifier) + { + try + { + return limsService.tryGetSampleWithExperiment(sampleIdentifier); + } catch (UserFailureException ex) + { + return null; + } + } + private void sendEmail(final String message, final ExperimentIdentifier experimentIdentifier, final String recipientMail) { diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java index 7b05e940bad..ad063278c3f 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultDataSetInfoExtractor.java @@ -328,7 +328,8 @@ public class DefaultDataSetInfoExtractor extends AbstractDataSetInfoExtractor } } - private List<NewProperty> extractDataSetProperties(File incomingDataSetPath, String fileName) + private static List<NewProperty> extractDataSetProperties(File incomingDataSetPath, + String fileName) { List<NewProperty> result = new ArrayList<NewProperty>(); if (fileName != null && incomingDataSetPath.isDirectory()) diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java index a7e3ed2bca1..c54e951f5fb 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java @@ -356,7 +356,8 @@ public final class ETLDaemon final TransferredDataSetHandler pathHandler = new TransferredDataSetHandler(dssCode, plugin, authorizedLimsService, mailProperties, highwaterMarkWatcher, notifySuccessfulRegistration, - threadParameters.useIsFinishedMarkerFile()); + threadParameters.useIsFinishedMarkerFile(), threadParameters + .deleteUnidentified()); pathHandler.setProcessorFactories(processorFactories); final HighwaterMarkDirectoryScanningHandler directoryScanningHandler = createDirectoryScanningHandler(pathHandler, highwaterMarkWatcher, diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java index 8475ddd5364..9e3978a5227 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ThreadParameters.java @@ -56,6 +56,8 @@ public final class ThreadParameters private static final String INCOMING_DIR = "incoming-dir"; + private static final String DELETE_UNIDENTIFIED_KEY = "delete-unidentified"; + /** * The (local) directory to monitor for new files and directories to move to the remote side. * The directory where data to be processed by the ETL server become available. @@ -70,6 +72,8 @@ public final class ThreadParameters private final boolean useIsFinishedMarkerFile; + private final boolean deleteUnidentified; + /** * @param threadProperties parameters for one processing thread together with general * parameters. @@ -83,6 +87,8 @@ public final class ThreadParameters PropertyUtils.getProperty(threadProperties, INCOMING_DATA_COMPLETENESS_CONDITION, INCOMING_DATA_COMPLETENESS_CONDITION_MARKER_FILE); this.useIsFinishedMarkerFile = parseCompletenessCondition(completenessCondition); + this.deleteUnidentified = + "true".equals(threadProperties.getProperty(DELETE_UNIDENTIFIED_KEY, "false")); this.threadName = threadName; } @@ -185,6 +191,7 @@ public final class ThreadParameters useIsFinishedMarkerFile ? "marker file exists" : "no write access for some period"; logLine("Condition of incoming data completeness: %s.", completenessCond); + logLine("Delete unidentified: '%s'.", deleteUnidentified); } } @@ -200,4 +207,9 @@ public final class ThreadParameters { return threadName; } + + public boolean deleteUnidentified() + { + return deleteUnidentified; + } } diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java index 3fe2c435101..6bd6aac1304 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java @@ -127,6 +127,8 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta private boolean stopped = false; + private boolean deleteUnidentified = false; + private Map<String, IProcessorFactory> processorFactories = Collections.<String, IProcessorFactory> emptyMap(); @@ -142,18 +144,20 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta public TransferredDataSetHandler(String dssCode, final IETLServerPlugin plugin, final IEncapsulatedOpenBISService limsService, final Properties mailProperties, final HighwaterMarkWatcher highwaterMarkWatcher, - final boolean notifySuccessfulRegistration, boolean useIsFinishedMarkerFile) + final boolean notifySuccessfulRegistration, boolean useIsFinishedMarkerFile, + boolean deleteUnidentified) { this(dssCode, plugin.getStorageProcessor(), plugin, limsService, new MailClient( - mailProperties), notifySuccessfulRegistration, useIsFinishedMarkerFile); + mailProperties), notifySuccessfulRegistration, useIsFinishedMarkerFile, + deleteUnidentified); } TransferredDataSetHandler(String dssCode, final IStoreRootDirectoryHolder storeRootDirectoryHolder, final IETLServerPlugin plugin, final IEncapsulatedOpenBISService limsService, final IMailClient mailClient, final boolean notifySuccessfulRegistration, - boolean useIsFinishedMarkerFile) + boolean useIsFinishedMarkerFile, boolean deleteUnidentified) { assert dssCode != null : "Unspecified data store code"; @@ -175,6 +179,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta this.registrationLock = new ReentrantLock(); this.fileOperations = FileOperations.getMonitoredInstanceForCurrentThread(); this.useIsFinishedMarkerFile = useIsFinishedMarkerFile; + this.deleteUnidentified = deleteUnidentified; } public final void setProcessorFactories(final Map<String, IProcessorFactory> processorFactories) @@ -497,6 +502,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta } final StopWatch watch = new StopWatch(); watch.start(); + ExternalData data = createExternalData(); File dataFile = storageProcessor.storeData(sample, dataSetInformation, typeExtractor, mailClient, incomingDataSetFile, baseDirectoryHolder @@ -518,7 +524,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta try { errorMessageTemplate = DATA_SET_REGISTRATION_FAILURE_TEMPLATE; - plainRegisterDataSet(relativePath, availableFormat, isCompleteFlag); + plainRegisterDataSet(data, relativePath, availableFormat, isCompleteFlag); clean(); clean(); if (processorOrNull == null) @@ -612,19 +618,19 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta final void moveDataSet() { final boolean ok = - FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder - .getTargetFile()); + deleteUnidentified ? (FileUtilities.deleteRecursively(incomingDataSetFile)) + : FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder + .getTargetFile()); if (ok) { clean(); } } - private final void plainRegisterDataSet(final String relativePath, + private final void plainRegisterDataSet(ExternalData data, final String relativePath, final StorageFormat storageFormat, final BooleanOrUnknown isCompleteFlag) { - final ExternalData data = - createExternalData(relativePath, storageFormat, isCompleteFlag); + updateExternalData(data, relativePath, storageFormat, isCompleteFlag); // Finally: register the data set in the database. limsService.registerDataSet(dataSetInformation, data); } @@ -763,18 +769,23 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta return null; } - private final ExternalData createExternalData(final String relativePath, + private final ExternalData updateExternalData(ExternalData data, final String relativePath, final StorageFormat storageFormat, final BooleanOrUnknown isCompleteFlag) + { + data.setComplete(isCompleteFlag); + data.setLocation(relativePath); + data.setStorageFormat(storageFormat); + return data; + } + + private ExternalData createExternalData() { final ExtractableData extractableData = dataSetInformation.getExtractableData(); final ExternalData data = BeanUtils.createBean(ExternalData.class, extractableData); - data.setLocation(relativePath); data.setLocatorType(typeExtractor.getLocatorType(incomingDataSetFile)); data.setDataSetType(typeExtractor.getDataSetType(incomingDataSetFile)); data.setFileFormatType(typeExtractor.getFileFormatType(incomingDataSetFile)); data.setMeasured(typeExtractor.isMeasuredData(incomingDataSetFile)); - data.setStorageFormat(storageFormat); - data.setComplete(isCompleteFlag); data.setDataStoreCode(dssCode); return data; } diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexDataSetInfoExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexDataSetInfoExtractor.java new file mode 100644 index 00000000000..afdc63b01c1 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexDataSetInfoExtractor.java @@ -0,0 +1,62 @@ +/* + * Copyright 2009 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.etlserver.cifex; + +import java.io.File; +import java.util.Properties; + +import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; +import ch.systemsx.cisd.common.exceptions.UserFailureException; +import ch.systemsx.cisd.etlserver.IDataSetInfoExtractor; +import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; +import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation; +import ch.systemsx.cisd.openbis.generic.shared.basic.DataSetUploadInfo; +import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier; +import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFactory; + +/** + * {@link IDataSetInfoExtractor} extracting group and sample from CIFEX comment. + * + * @author Izabela Adamczyk + */ +public class CifexDataSetInfoExtractor implements IDataSetInfoExtractor +{ + + public CifexDataSetInfoExtractor(final Properties globalProperties) + { + } + + public DataSetInformation getDataSetInformation(File incomingDataSetPath, + IEncapsulatedOpenBISService openbisService) throws UserFailureException, + EnvironmentFailureException + { + assert incomingDataSetPath != null : "Incoming data set path can not be null."; + + DataSetUploadInfo info = CifexExtratorHelper.getDataSetUploadInfo(incomingDataSetPath); + final DataSetInformation dataSetInformation = new DataSetInformation(); + SampleIdentifier identifier = SampleIdentifierFactory.parse(info.getSample()); + dataSetInformation.setSampleCode(identifier.getSampleCode()); + if (identifier.isGroupLevel()) + { + dataSetInformation.setGroupCode(identifier.getGroupLevel().getGroupCode()); + } + dataSetInformation.setUploadingUserEmail(CifexExtratorHelper + .getUploadingUserEmail(incomingDataSetPath)); + return dataSetInformation; + } + +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexExtratorHelper.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexExtratorHelper.java new file mode 100644 index 00000000000..201fa22e642 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexExtratorHelper.java @@ -0,0 +1,64 @@ +/* + * Copyright 2009 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.etlserver.cifex; + +import java.io.File; +import java.util.Properties; + +import ch.systemsx.cisd.common.exceptions.UserFailureException; +import ch.systemsx.cisd.common.utilities.PropertyUtils; +import ch.systemsx.cisd.openbis.generic.shared.basic.DataSetUploadInfo; +import ch.systemsx.cisd.openbis.generic.shared.basic.DataSetUploadInfo.DataSetUploadInfoHelper; + +/** + * @author Izabela Adamczyk + */ +public class CifexExtratorHelper +{ + private static final String COMMENT_KEY = "comment"; + + private static final String REQUEST_PROPERTIES_FILE = "request.properties"; + + private static final String UPLOADING_USER_EMAIL_KEY = "user-email"; + + public static DataSetUploadInfo getDataSetUploadInfo(File incomingDataSetPath) + { + Properties properties = loadProperties(incomingDataSetPath, REQUEST_PROPERTIES_FILE); + String comment = properties.getProperty(COMMENT_KEY); + DataSetUploadInfo info = DataSetUploadInfoHelper.extractFromCifexComment(comment); + return info; + } + + public static String getUploadingUserEmail(File incomingDataSetPath) + { + Properties properties = loadProperties(incomingDataSetPath, REQUEST_PROPERTIES_FILE); + return properties.getProperty(UPLOADING_USER_EMAIL_KEY); + } + + private static Properties loadProperties(File incomingDataSetPath, String fileName) + { + File propertiesFile = new File(incomingDataSetPath, fileName); + if (propertiesFile.isFile()) + { + return PropertyUtils.loadProperties(propertiesFile.getPath()); + } else + { + throw new UserFailureException("Request properties file '" + propertiesFile + + "' does not exist or is not a 'normal' file."); + } + } +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexTypeExtractor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexTypeExtractor.java new file mode 100644 index 00000000000..0e72f721113 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/cifex/CifexTypeExtractor.java @@ -0,0 +1,73 @@ +/* + * Copyright 2009 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.etlserver.cifex; + +import java.io.File; +import java.util.Properties; + +import ch.systemsx.cisd.etlserver.ITypeExtractor; +import ch.systemsx.cisd.etlserver.SimpleTypeExtractor; +import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetType; +import ch.systemsx.cisd.openbis.generic.shared.dto.FileFormatType; +import ch.systemsx.cisd.openbis.generic.shared.dto.LocatorType; + +/** + * {@link ITypeExtractor} which extracts data set type and file format from the CIFEX comment saved + * in 'request.properties' file. <br> + * Locator type, processor type and 'is measured' values are calculated by corresponding methods of + * {@link SimpleTypeExtractor}. + * + * @author Izabela Adamczyk + */ +public class CifexTypeExtractor implements ITypeExtractor +{ + + private ITypeExtractor simpleTypeExtractor; + + public CifexTypeExtractor(final Properties properties) + { + simpleTypeExtractor = new SimpleTypeExtractor(properties); + } + + public DataSetType getDataSetType(File incomingDataSetPath) + { + return new DataSetType(CifexExtratorHelper.getDataSetUploadInfo(incomingDataSetPath) + .getDataSetType()); + } + + public FileFormatType getFileFormatType(File incomingDataSetPath) + { + return new FileFormatType(CifexExtratorHelper.getDataSetUploadInfo(incomingDataSetPath) + .getFileType()); + } + + public LocatorType getLocatorType(File incomingDataSetPath) + { + return simpleTypeExtractor.getLocatorType(incomingDataSetPath); + } + + public String getProcessorType(File incomingDataSetPath) + { + return simpleTypeExtractor.getProcessorType(incomingDataSetPath); + } + + public boolean isMeasuredData(File incomingDataSetPath) + { + return simpleTypeExtractor.isMeasuredData(incomingDataSetPath); + } + +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java index ea2218e4c86..643c3c26a0a 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java @@ -82,11 +82,26 @@ public class DataSetInformation implements Serializable */ private ExtractableData extractableData = new ExtractableData(); + /** + * Email of uploading user. + */ + private String uploadingUserEmailOrNull; + /** This constructor is for serialization. */ public DataSetInformation() { } + public String tryGetUploadingUserEmail() + { + return uploadingUserEmailOrNull; + } + + public void setUploadingUserEmail(String uploadingUserEmail) + { + this.uploadingUserEmailOrNull = uploadingUserEmail; + } + public final BooleanOrUnknown getIsCompleteFlag() { return isCompleteFlag; diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java index ac6c6b57364..4c6681fb665 100644 --- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java +++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java @@ -277,7 +277,7 @@ public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestC authorizedLimsService.setPassword("p"); handler = new TransferredDataSetHandler("dss", storageProcessor, plugin, - authorizedLimsService, mailClient, true, true); + authorizedLimsService, mailClient, true, true, false); handler.setProcessorFactories(map); dataSetInformation = new DataSetInformation(); @@ -842,6 +842,11 @@ public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestC will(throwException(new Exception("Could store data by storage processor"))); one(storageProcessor).unstoreData(with(equal(folder)), with(equal(baseDir))); + + allowing(typeExtractor).getLocatorType(folder); + allowing(typeExtractor).getDataSetType(folder); + allowing(typeExtractor).getFileFormatType(folder); + allowing(typeExtractor).isMeasuredData(folder); } }); final LogMonitoringAppender appender = -- GitLab