From b9d2e8df01e89bd8851716ab910e4c0d42c24417 Mon Sep 17 00:00:00 2001 From: cramakri <cramakri> Date: Wed, 16 Feb 2011 09:46:36 +0000 Subject: [PATCH] LMS-1988 Improvements to the service/transaction interface. SVN: 20002 --- ...tOmniscientTopLevelDataSetRegistrator.java | 10 +++- .../DataSetRegistrationDetails.java | 5 ++ .../DataSetRegistrationService.java | 58 ++++++++++++++++--- .../registrator/DataSetStorageAlgorithm.java | 33 +++++++---- .../IDataSetRegistrationService.java | 51 ++++++++++++++++ .../JythonTopLevelDataSetHandler.java | 14 +++-- .../registrator/api/v1/impl/DataSet.java | 3 +- .../JythonTopLevelDataSetRegistratorTest.java | 10 ++-- .../DataSetRegistrationTransactionTest.java | 7 +-- 9 files changed, 153 insertions(+), 38 deletions(-) create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetRegistrationService.java diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java index 9a452c0525f..9f2ad6f09e7 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/AbstractOmniscientTopLevelDataSetRegistrator.java @@ -235,7 +235,7 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat } DataSetRegistrationService<T> service = - createDataSetRegistrationService(cleanAfterwardsAction); + createDataSetRegistrationService(incomingDataSetFile, cleanAfterwardsAction); try { @@ -311,14 +311,18 @@ public abstract class AbstractOmniscientTopLevelDataSetRegistrator<T extends Dat /** * Create the data set registration service. + * + * @param incomingDataSetFile */ protected DataSetRegistrationService<T> createDataSetRegistrationService( + File incomingDataSetFile, final IDelegatedActionWithResult<Boolean> cleanAfterwardsAction) { @SuppressWarnings("unchecked") DataSetRegistrationService<T> service = - new DataSetRegistrationService(this, new DefaultDataSetRegistrationDetailsFactory( - getRegistratorState()), cleanAfterwardsAction); + new DataSetRegistrationService(this, incomingDataSetFile, + new DefaultDataSetRegistrationDetailsFactory(getRegistratorState()), + cleanAfterwardsAction); return service; } diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationDetails.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationDetails.java index bc884f697cb..dc738efc743 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationDetails.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationDetails.java @@ -119,6 +119,11 @@ public class DataSetRegistrationDetails<T extends DataSetInformation> implements dataSetInformation.setDataSetType(dataSetType); } + public void setDataSetType(String dataSetTypeCode) + { + setDataSetType(new DataSetType(dataSetTypeCode)); + } + public T getDataSetInformation() { return dataSetInformation; diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java index 6d4fbb11261..550f045ba60 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetRegistrationService.java @@ -22,16 +22,20 @@ import java.util.Properties; import ch.systemsx.cisd.common.utilities.IDelegatedActionWithResult; import ch.systemsx.cisd.common.utilities.PropertyUtils; +import ch.systemsx.cisd.etlserver.BaseDirectoryHolder; import ch.systemsx.cisd.etlserver.DataSetRegistrationAlgorithm; import ch.systemsx.cisd.etlserver.DataSetRegistrationAlgorithm.DataSetRegistrationAlgorithmState; import ch.systemsx.cisd.etlserver.DataSetRegistrationAlgorithmRunner; +import ch.systemsx.cisd.etlserver.FileRenamer; import ch.systemsx.cisd.etlserver.IDataStoreStrategy; import ch.systemsx.cisd.etlserver.IdentifiedDataStrategy; import ch.systemsx.cisd.etlserver.TopLevelDataSetRegistratorGlobalState; +import ch.systemsx.cisd.etlserver.TransferredDataSetHandler; import ch.systemsx.cisd.etlserver.registrator.AbstractOmniscientTopLevelDataSetRegistrator.OmniscientTopLevelDataSetRegistratorState; import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSetRegistrationTransaction; import ch.systemsx.cisd.etlserver.registrator.api.v1.impl.DataSetRegistrationTransaction; import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType; import ch.systemsx.cisd.openbis.generic.shared.dto.NewExternalData; /** @@ -40,7 +44,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.NewExternalData; * @author Chandrasekhar Ramakrishnan */ public class DataSetRegistrationService<T extends DataSetInformation> implements - DataSetRegistrationAlgorithm.IRollbackDelegate + DataSetRegistrationAlgorithm.IRollbackDelegate, IDataSetRegistrationService { static final String STAGING_DIR = "staging-dir"; @@ -57,6 +61,8 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements private final File stagingDirectory; + private final File incomingDataSetFile; + /** * All transactions ever created on this service. */ @@ -89,11 +95,13 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements * @param globalCleanAfterwardsAction An action to execute when the service has finished */ public DataSetRegistrationService(AbstractOmniscientTopLevelDataSetRegistrator<T> registrator, + File incomingDataSetFile, IDataSetRegistrationDetailsFactory<T> registrationDetailsFactory, IDelegatedActionWithResult<Boolean> globalCleanAfterwardsAction) { this.registrator = registrator; this.registratorContext = registrator.getRegistratorState(); + this.incomingDataSetFile = incomingDataSetFile; this.globalCleanAfterwardsAction = globalCleanAfterwardsAction; this.dataSetRegistrationDetailsFactory = registrationDetailsFactory; @@ -134,14 +142,17 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements /** * Create a new transaction that atomically performs file operations and registers entities. */ - public IDataSetRegistrationTransaction transaction(File dataSetFile) + public IDataSetRegistrationTransaction transaction() { - return transaction(dataSetFile, getDataSetRegistrationDetailsFactory()); + return transaction(incomingDataSetFile, getDataSetRegistrationDetailsFactory()); } - protected IDataSetRegistrationDetailsFactory<T> getDataSetRegistrationDetailsFactory() + /** + * Create a new transaction that atomically performs file operations and registers entities. + */ + public IDataSetRegistrationTransaction transaction(File dataSetFile) { - return dataSetRegistrationDetailsFactory; + return transaction(dataSetFile, getDataSetRegistrationDetailsFactory()); } /** @@ -186,6 +197,32 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements dataSetRegistrations.clear(); } + public File moveIncomingToError(DataSetRegistrationDetails<?> dataSetDetails) + { + // Make sure the data set information is valid + DataSetInformation dataSetInfo = dataSetDetails.getDataSetInformation(); + dataSetInfo.setShareId(registratorContext.getGlobalState().getShareId()); + if (null == dataSetInfo.getDataSetType()) + { + dataSetInfo.setDataSetType(new DataSetType("UNKNOWN")); + } + + // Create the error directory + File baseDirectory = + DataSetStorageAlgorithm.createBaseDirectory( + TransferredDataSetHandler.ERROR_DATA_STRATEGY, registratorContext + .getStorageProcessor().getStoreRootDirectory(), registratorContext + .getFileOperations(), dataSetInfo, dataSetDetails.getDataSetType(), + incomingDataSetFile); + BaseDirectoryHolder baseDirectoryHolder = + new BaseDirectoryHolder(TransferredDataSetHandler.ERROR_DATA_STRATEGY, + baseDirectory, incomingDataSetFile); + + // Move the incoming there + FileRenamer.renameAndLog(incomingDataSetFile, baseDirectoryHolder.getTargetFile()); + return baseDirectoryHolder.getTargetFile(); + } + public void rollbackTransaction(DataSetRegistrationTransaction<T> transaction, DataSetStorageAlgorithmRunner<T> algorithm, Throwable ex) { @@ -241,6 +278,11 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements return new DefaultEntityOperationService<T>(registrator); } + protected IDataSetRegistrationDetailsFactory<T> getDataSetRegistrationDetailsFactory() + { + return dataSetRegistrationDetailsFactory; + } + /** * If a transaction is hanging around, commit it */ @@ -268,7 +310,7 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements } } - private DataSetRegistrationAlgorithm createRegistrationAlgorithm(File incomingDataSetFile, + private DataSetRegistrationAlgorithm createRegistrationAlgorithm(File aDataSetFile, DataSetRegistrationDetails<T> details) { final TopLevelDataSetRegistratorGlobalState globalState = @@ -285,10 +327,10 @@ public class DataSetRegistrationService<T extends DataSetInformation> implements IDataStoreStrategy dataStoreStrategy = registratorContext.getDataStrategyStore().getDataStoreStrategy( - details.getDataSetInformation(), incomingDataSetFile); + details.getDataSetInformation(), aDataSetFile); DataSetRegistrationAlgorithmState state = - new DataSetRegistrationAlgorithmState(incomingDataSetFile, + new DataSetRegistrationAlgorithmState(aDataSetFile, globalState.getOpenBisService(), cleanAfterwardsAction, registratorContext.getPreRegistrationAction(), registratorContext.getPostRegistrationAction(), diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithm.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithm.java index 3a229d84724..b3968bb6d62 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithm.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/DataSetStorageAlgorithm.java @@ -98,6 +98,24 @@ public class DataSetStorageAlgorithm<T extends DataSetInformation> // State that changes during execution private DataSetStorageAlgorithmState<T> state; + /** + * Utility method for creating base directories + */ + public static File createBaseDirectory(final IDataStoreStrategy strategy, final File baseDir, + IFileOperations fileOperations, final DataSetInformation dataSetInfo, + DataSetType dataSetType, File incomingDataSetFile) + { + final File baseDirectory = strategy.getBaseDirectory(baseDir, dataSetInfo, dataSetType); + baseDirectory.mkdirs(); + if (fileOperations.isDirectory(baseDirectory) == false) + { + throw EnvironmentFailureException.fromTemplate( + "Creating data set base directory '%s' for data set '%s' failed.", + baseDirectory.getAbsolutePath(), incomingDataSetFile); + } + return baseDirectory; + } + public DataSetStorageAlgorithm(File incomingDataSetFile, DataSetRegistrationDetails<T> registrationDetails, IDataStoreStrategy dataStoreStrategy, IStorageProcessor storageProcessor, @@ -297,17 +315,10 @@ public class DataSetStorageAlgorithm<T extends DataSetInformation> protected final File createBaseDirectory(final IDataStoreStrategy strategy, final File baseDir, final DataSetInformation dataSetInfo) { - final File baseDirectory = - strategy.getBaseDirectory(baseDir, dataSetInfo, - storageAlgorithm.getDataSetType()); - baseDirectory.mkdirs(); - if (getFileOperations().isDirectory(baseDirectory) == false) - { - throw EnvironmentFailureException.fromTemplate( - "Creating data set base directory '%s' for data set '%s' failed.", - baseDirectory.getAbsolutePath(), incomingDataSetFile); - } - return baseDirectory; + + return DataSetStorageAlgorithm.createBaseDirectory(strategy, baseDir, + getFileOperations(), dataSetInfo, storageAlgorithm.getDataSetType(), + incomingDataSetFile); } } diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetRegistrationService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetRegistrationService.java new file mode 100644 index 00000000000..6ba191cf107 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/IDataSetRegistrationService.java @@ -0,0 +1,51 @@ +/* + * Copyright 2011 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.etlserver.registrator; + +import java.io.File; + +import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSetRegistrationTransaction; + +/** + * @author Chandrasekhar Ramakrishnan + */ +public interface IDataSetRegistrationService +{ + + /** + * Create a new transaction that atomically performs file operations and registers entities, + * using the file as the root for resolving relative paths. + * + * @return A new transaction + */ + public IDataSetRegistrationTransaction transaction(); + + /** + * Create a new transaction that atomically performs file operations and registers entities, + * using the file as the root for resolving relative paths. + * + * @param dataSetFile The file for resolving relative paths. + * @return A new transaction + */ + public IDataSetRegistrationTransaction transaction(File dataSetFile); + + /** + * Move the incoming file to the error directory. + */ + public File moveIncomingToError(DataSetRegistrationDetails<?> dataSetDetails); + +} \ No newline at end of file diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java index 20b181e5d8a..67cd6aa2d77 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetHandler.java @@ -136,12 +136,13 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends */ @Override protected DataSetRegistrationService<T> createDataSetRegistrationService( - IDelegatedActionWithResult<Boolean> cleanAfterwardsAction) + File incomingDataSetFile, IDelegatedActionWithResult<Boolean> cleanAfterwardsAction) { PythonInterpreter interpreter = new PythonInterpreter(); interpreter.set(STATE_VARIABLE_NAME, getGlobalState()); JythonDataSetRegistrationService<T> service = - createJythonDataSetRegistrationService(cleanAfterwardsAction, interpreter); + createJythonDataSetRegistrationService(incomingDataSetFile, cleanAfterwardsAction, + interpreter); return service; } @@ -149,10 +150,12 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends * Create a Jython registration service that includes access to the interpreter. */ protected JythonDataSetRegistrationService<T> createJythonDataSetRegistrationService( - IDelegatedActionWithResult<Boolean> cleanAfterwardsAction, PythonInterpreter interpreter) + File incomingDataSetFile, IDelegatedActionWithResult<Boolean> cleanAfterwardsAction, + PythonInterpreter interpreter) { JythonDataSetRegistrationService<T> service = - new JythonDataSetRegistrationService<T>(this, cleanAfterwardsAction, interpreter); + new JythonDataSetRegistrationService<T>(this, incomingDataSetFile, + cleanAfterwardsAction, interpreter); return service; } @@ -289,10 +292,11 @@ public class JythonTopLevelDataSetHandler<T extends DataSetInformation> extends * @param globalCleanAfterwardsAction */ public JythonDataSetRegistrationService(JythonTopLevelDataSetHandler<T> registrator, + File incomingDataSetFile, IDelegatedActionWithResult<Boolean> globalCleanAfterwardsAction, PythonInterpreter interpreter) { - super(registrator, registrator.createObjectFactory(interpreter), + super(registrator, incomingDataSetFile, registrator.createObjectFactory(interpreter), globalCleanAfterwardsAction); this.interpreter = interpreter; } diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSet.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSet.java index 3de83b0cbab..eaa188ba1ca 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSet.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSet.java @@ -24,7 +24,6 @@ import ch.systemsx.cisd.etlserver.registrator.api.v1.IDataSet; import ch.systemsx.cisd.etlserver.registrator.api.v1.IExperimentImmutable; import ch.systemsx.cisd.etlserver.registrator.api.v1.ISampleImmutable; import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation; -import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.FileFormatType; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space; import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier; @@ -151,7 +150,7 @@ public class DataSet<T extends DataSetInformation> implements IDataSet public void setDataSetType(String dataSetTypeCode) { - registrationDetails.setDataSetType(new DataSetType(dataSetTypeCode)); + registrationDetails.setDataSetType(dataSetTypeCode); } protected void setExperiment(ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment exp) diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java index d2672673a3a..154c500e288 100644 --- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java +++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/JythonTopLevelDataSetRegistratorTest.java @@ -1064,12 +1064,12 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractFileSystemTest @Override protected JythonDataSetRegistrationService<DataSetInformation> createJythonDataSetRegistrationService( - IDelegatedActionWithResult<Boolean> cleanAfterwardsAction, + File aDataSetFile, IDelegatedActionWithResult<Boolean> cleanAfterwardsAction, PythonInterpreter interpreter) { JythonDataSetRegistrationService<DataSetInformation> service = - new TestDataRegistrationService(this, cleanAfterwardsAction, interpreter, - shouldRegistrationFail); + new TestDataRegistrationService(this, aDataSetFile, cleanAfterwardsAction, + interpreter, shouldRegistrationFail); return service; } @@ -1086,11 +1086,11 @@ public class JythonTopLevelDataSetRegistratorTest extends AbstractFileSystemTest * @param interpreter */ public TestDataRegistrationService( - JythonTopLevelDataSetHandler<DataSetInformation> registrator, + JythonTopLevelDataSetHandler<DataSetInformation> registrator, File aDataSetFile, IDelegatedActionWithResult<Boolean> globalCleanAfterwardsAction, PythonInterpreter interpreter, boolean shouldRegistrationFail) { - super(registrator, globalCleanAfterwardsAction, interpreter); + super(registrator, aDataSetFile, globalCleanAfterwardsAction, interpreter); this.shouldRegistrationFail = shouldRegistrationFail; } diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSetRegistrationTransactionTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSetRegistrationTransactionTest.java index 0e830faec24..e7aedf26333 100644 --- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSetRegistrationTransactionTest.java +++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSetRegistrationTransactionTest.java @@ -302,7 +302,7 @@ public class DataSetRegistrationTransactionTest extends AbstractFileSystemTestCa TopLevelDataSetRegistratorGlobalState globalState = createGlobalState(threadProperties); handler = new TestingDataSetHandler(globalState); - service = handler.createNewService(); + service = handler.createNewService(workingDirectory); } private TopLevelDataSetRegistratorGlobalState createGlobalState(Properties threadProperties) @@ -373,7 +373,6 @@ public class DataSetRegistrationTransactionTest extends AbstractFileSystemTestCa }); } - private void setUpDataSetValidatorExpectations() { File dataSetDir = new File(stagingDirectory, DATA_SET_CODE + "1"); @@ -481,7 +480,7 @@ public class DataSetRegistrationTransactionTest extends AbstractFileSystemTestCa } - public DataSetRegistrationService<DataSetInformation> createNewService() + public DataSetRegistrationService<DataSetInformation> createNewService(File dataSetFile) { IDelegatedActionWithResult<Boolean> cleanAfterwardsAction = new IDelegatedActionWithResult<Boolean>() @@ -491,7 +490,7 @@ public class DataSetRegistrationTransactionTest extends AbstractFileSystemTestCa return true; // do nothing } }; - return createDataSetRegistrationService(cleanAfterwardsAction); + return createDataSetRegistrationService(dataSetFile, cleanAfterwardsAction); } /** -- GitLab