Skip to content
Snippets Groups Projects
Commit 37138d76 authored by cramakri's avatar cramakri
Browse files

LMS-1811 Updated CinaBundleDataSetHandler and its test. 

SVN: 18285
parent 5b6345e8
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 549 additions and 906 deletions
rtd_cina/source/java/ch/systemsx/cisd/cina/dss/bundle/BundleDataSetHelper.java deleted 100644 → 0
+ 0
− 119
View file @ 5b6345e8
/*
* Copyright 2010 ETH Zuerich, CISD
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.systemsx.cisd.cina.dss.bundle;
import java.io.File;
import java.util.ArrayList;
import java.util.List;
import ch.systemsx.cisd.etlserver.IDataSetHandler;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetTypeWithVocabularyTerms;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
/**
* Helper to help registering bundle data sets.
* <p>
* This helper registers the entire bundle as one data set.
*
* @author Chandrasekhar Ramakrishnan
*/
class BundleDataSetHelper
{
static class BundleRegistrationGlobalState
{
private final IDataSetHandler delegator;
private final IEncapsulatedOpenBISService openbisService;
private final SampleType replicaSampleType;
private final DataSetTypeWithVocabularyTerms imageDataSetType;
BundleRegistrationGlobalState(IDataSetHandler delegator,
IEncapsulatedOpenBISService openbisService, SampleType replicaSampleType,
DataSetTypeWithVocabularyTerms imageDataSetType)
{
this.delegator = delegator;
this.openbisService = openbisService;
this.replicaSampleType = replicaSampleType;
this.imageDataSetType = imageDataSetType;
}
IDataSetHandler getDelegator()
{
return delegator;
}
IEncapsulatedOpenBISService getOpenbisService()
{
return openbisService;
}
SampleType getReplicaSampleType()
{
return replicaSampleType;
}
DataSetTypeWithVocabularyTerms getImageDataSetType()
{
return imageDataSetType;
}
}
protected final BundleRegistrationGlobalState globalState;
protected final File dataSet;
protected final ArrayList<DataSetInformation> dataSetInformation;
BundleDataSetHelper(BundleRegistrationGlobalState globalState, File dataSet)
{
this.globalState = globalState;
this.dataSet = dataSet;
this.dataSetInformation = new ArrayList<DataSetInformation>();
}
public void process()
{
// Register the bundle as one data set
List<DataSetInformation> bigDataSet = getDelegator().handleDataSet(dataSet);
dataSetInformation.addAll(bigDataSet);
}
protected IDataSetHandler getDelegator()
{
return globalState.getDelegator();
}
protected IEncapsulatedOpenBISService getOpenbisService()
{
return globalState.getOpenbisService();
}
/**
* Get all the data set information that has been created as a result of {@link #process}. Only
* makes sense to invoke after process has been called
*/
public ArrayList<DataSetInformation> getDataSetInformation()
{
return dataSetInformation;
}
}
rtd_cina/source/java/ch/systemsx/cisd/cina/dss/bundle/BundleDataSetHelperRpc.java deleted 100644 → 0
+ 0
− 211
View file @ 5b6345e8
/*
* Copyright 2010 ETH Zuerich, CISD
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.systemsx.cisd.cina.dss.bundle;
import java.io.File;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
import ch.systemsx.cisd.cina.shared.metadata.BundleMetadataExtractor;
import ch.systemsx.cisd.cina.shared.metadata.ImageMetadataExtractor;
import ch.systemsx.cisd.cina.shared.metadata.ReplicaMetadataExtractor;
import ch.systemsx.cisd.etlserver.IDataSetHandler;
import ch.systemsx.cisd.etlserver.IDataSetHandlerRpc;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewSample;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PropertyTypeWithVocabulary;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
import ch.systemsx.cisd.openbis.generic.shared.dto.NewProperty;
import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SpaceIdentifier;
/**
* Helper class for processing bundle data sets.
* <p>
* This helper registers the entire bundle as one data set, generates samples for each replica, and
* registers each file as a data set associated with the sample.
*
* @author Chandrasekhar Ramakrishnan
*/
class BundleDataSetHelperRpc extends BundleDataSetHelper
{
private static final String MISC_DATA_SET_PROPERTY_CODE = "MISC";
// Invocation-specific State
private final SessionContextDTO sessionContext;
private final IDataSetHandlerRpc delegatorRpc;
BundleDataSetHelperRpc(BundleRegistrationGlobalState globalState, File dataSet)
{
super(globalState, dataSet);
IDataSetHandler delegator = globalState.getDelegator();
if (delegator instanceof IDataSetHandlerRpc)
{
delegatorRpc = (IDataSetHandlerRpc) delegator;
this.sessionContext = delegatorRpc.getSessionContext();
} else
{
throw new IllegalArgumentException(
"BundleDataSetHelperRpc used with non-RPC data set handler.");
}
}
/**
*
*/
@Override
public void process()
{
String dataSetName = dataSet.getName();
// Register the bundle as one data set
super.process();
if (dataSetInformation.isEmpty())
{
return;
}
ExternalData bigData =
getOpenbisService().tryGetDataSet(sessionContext.getSessionToken(),
getBigDataSetInformation().getDataSetCode());
File containerFile = delegatorRpc.getFileForExternalData(bigData);
File bundle = new File(containerFile, dataSetName);
BundleMetadataExtractor bundleMetadata = new BundleMetadataExtractor(bundle);
bundleMetadata.prepare();
for (ReplicaMetadataExtractor replicaMetadata : bundleMetadata.getReplicaMetadataExtractors())
{
handleDerivedDataSets(replicaMetadata);
}
}
private void handleDerivedDataSets(ReplicaMetadataExtractor replicaMetadata)
{
SampleIdentifier sampleId = registerReplicaSample();
// Register all the data sets derived from this sample
for (ImageMetadataExtractor imageMetadata : replicaMetadata.getImageMetadataExtractors())
{
handleDerivedDataSet(sampleId, imageMetadata);
}
}
/**
* Register a sample with this ID of type GRID_REPLICA, either derived from the GRID_TEMPLATE
* registered by the client or associated with the experiment.
*/
private SampleIdentifier registerReplicaSample()
{
DataSetInformation bigDataSetInfo = getBigDataSetInformation();
SampleType replicaSampleType = globalState.getReplicaSampleType();
long sampleCodeSuffix = getOpenbisService().drawANewUniqueID();
String sampleCode =
String.format("%s%d", replicaSampleType.getGeneratedCodePrefix(), sampleCodeSuffix);
ExperimentIdentifier experimentIdOrNull = bigDataSetInfo.getExperimentIdentifier();
SampleIdentifier parentIdOrNull = bigDataSetInfo.getSampleIdentifier();
// Either the experimentId or the parentId must be non-null
assert experimentIdOrNull != null || parentIdOrNull != null;
SampleIdentifier sampleId = null;
NewSample sample = null;
if (parentIdOrNull != null)
{
Sample parentSample = getOpenbisService().tryGetSampleWithExperiment(parentIdOrNull);
sampleId = new SampleIdentifier(parentIdOrNull.getSpaceLevel(), sampleCode);
sample =
NewSample.createWithParent(sampleId.toString(), replicaSampleType, null,
parentIdOrNull.toString());
sample.setExperimentIdentifier(parentSample.getExperiment().getIdentifier());
} else if (experimentIdOrNull != null)
{
SpaceIdentifier spaceId =
new SpaceIdentifier(experimentIdOrNull.getDatabaseInstanceCode(),
experimentIdOrNull.getSpaceCode());
sampleId = new SampleIdentifier(spaceId, sampleCode);
sample = NewSample.createWithParent(sampleId.toString(), replicaSampleType, null, null);
sample.setExperimentIdentifier(experimentIdOrNull.toString());
}
String userIdOrNull = (sessionContext == null) ? null : sessionContext.getUserName();
getOpenbisService().registerSample(sample, userIdOrNull);
return sampleId;
}
private void handleDerivedDataSet(SampleIdentifier sampleId,
ImageMetadataExtractor imageMetadata)
{
// Create a DataSetInformation
DataSetInformation imageDataSetInfo = createDataSetInformation(sampleId);
// Import the metadata
ArrayList<NewProperty> properties = createDataSetProperties(imageMetadata);
imageDataSetInfo.setDataSetProperties(properties);
File imageDataSet = imageMetadata.getFolder();
delegatorRpc.linkAndHandleDataSet(imageDataSet, imageDataSetInfo);
}
private ArrayList<NewProperty> createDataSetProperties(ImageMetadataExtractor imageMetadata)
{
List<PropertyTypeWithVocabulary> propertyTypes =
globalState.getImageDataSetType().getPropertyTypes();
ArrayList<NewProperty> properties = new ArrayList<NewProperty>();
for (PropertyTypeWithVocabulary propertyType : propertyTypes)
{
String value = imageMetadata.getMetadataMap().get(propertyType.getCode().toLowerCase());
if (null != value)
{
NewProperty prop;
prop = new NewProperty(propertyType.getCode(), value);
properties.add(prop);
}
}
// Add a property with everything
properties.add(new NewProperty(MISC_DATA_SET_PROPERTY_CODE, imageMetadata.getMetadataMap()
.toString()));
return properties;
}
private DataSetInformation createDataSetInformation(SampleIdentifier sampleId)
{
DataSetInformation imageDataSetInfo = new DataSetInformation();
imageDataSetInfo.setSampleCode(sampleId.getSampleCode());
imageDataSetInfo.setSpaceCode(sampleId.getSpaceLevel().getSpaceCode());
imageDataSetInfo.setInstanceCode(sampleId.getSpaceLevel().getDatabaseInstanceCode());
imageDataSetInfo.setDataSetType(globalState.getImageDataSetType().getDataSetType());
imageDataSetInfo.setParentDataSetCodes(Collections.singletonList(getBigDataSetInformation()
.getDataSetCode()));
return imageDataSetInfo;
}
/**
* Get the "big data set" -- the one that is the parent of all the derived data sets I register
*/
private DataSetInformation getBigDataSetInformation()
{
// The big data set is the first one registered.
return dataSetInformation.get(0);
}
}
rtd_cina/source/java/ch/systemsx/cisd/cina/dss/bundle/CinaBundleDataSetHandler.java
+ 18
− 18
View file @ 37138d76
......@@ -20,14 +20,12 @@ import java.io.File;
import java.util.List;
import java.util.Properties;
import ch.systemsx.cisd.cina.dss.bundle.BundleDataSetHelper.BundleRegistrationGlobalState;
import ch.systemsx.cisd.cina.shared.constants.CinaConstants;
import ch.systemsx.cisd.cina.dss.bundle.registrators.BundleRegistrationState;
import ch.systemsx.cisd.cina.dss.bundle.registrators.GridPreparationRegistrator;
import ch.systemsx.cisd.etlserver.IDataSetHandler;
import ch.systemsx.cisd.etlserver.IDataSetHandlerRpc;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetTypeWithVocabularyTerms;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
/**
* CINA registers data in the form of a bundle. A bundle contains several different kinds data sets.
......@@ -84,23 +82,30 @@ public class CinaBundleDataSetHandler implements IDataSetHandler
{
private final IDataSetHandler delegator;
private final BundleRegistrationGlobalState bundleRegistrationState;
private final BundleRegistrationState bundleRegistrationState;
public CinaBundleDataSetHandler(Properties parentProperties, IDataSetHandler delegator,
IEncapsulatedOpenBISService openbisService)
{
this.delegator = delegator;
this.bundleRegistrationState = createBundleRegistrationState(delegator, openbisService);
if (delegator instanceof IDataSetHandlerRpc)
{
this.bundleRegistrationState =
createBundleRegistrationState((IDataSetHandlerRpc) delegator, openbisService);
} else
{
this.bundleRegistrationState = null;
}
}
public List<DataSetInformation> handleDataSet(File dataSet)
{
BundleDataSetHelper helper;
if (delegator instanceof IDataSetHandlerRpc)
{
helper = new BundleDataSetHelperRpc(bundleRegistrationState, dataSet);
helper.process();
return helper.getDataSetInformation();
GridPreparationRegistrator registrator =
new GridPreparationRegistrator(bundleRegistrationState, dataSet);
registrator.register();
return registrator.getDataSetInformation();
} else
{
// We are not being invoked from the command line, so we don't have enough contextual
......@@ -110,14 +115,9 @@ public class CinaBundleDataSetHandler implements IDataSetHandler
}
}
private static BundleRegistrationGlobalState createBundleRegistrationState(
IDataSetHandler delegator, IEncapsulatedOpenBISService openbisService)
private static BundleRegistrationState createBundleRegistrationState(
IDataSetHandlerRpc delegator, IEncapsulatedOpenBISService openbisService)
{
SampleType replicaSampleType =
openbisService.getSampleType(CinaConstants.REPLICA_SAMPLE_TYPE_CODE);
DataSetTypeWithVocabularyTerms imageDataSetType =
openbisService.getDataSetType(CinaConstants.IMAGE_DATA_SET_TYPE_CODE);
return new BundleRegistrationGlobalState(delegator, openbisService, replicaSampleType,
imageDataSetType);
return new BundleRegistrationState(delegator, openbisService);
}
}
rtd_cina/sourceTest/java/ch/systemsx/cisd/cina/dss/bundle/CinaBundleDataSetHandlerTest.java
+ 36
− 138
View file @ 37138d76
......@@ -17,104 +17,66 @@
package ch.systemsx.cisd.cina.dss.bundle;
import java.io.File;
import java.io.IOException;
import java.util.Collections;
import java.util.Properties;
import org.jmock.Expectations;
import org.jmock.Mockery;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
import ch.systemsx.cisd.cina.shared.constants.CinaConstants;
import ch.systemsx.cisd.etlserver.IDataSetHandlerRpc;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetTypeWithVocabularyTerms;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewSample;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO;
import ch.systemsx.cisd.cina.dss.bundle.registrators.CinaBundleRegistrationTest;
/**
* @author Chandrasekhar Ramakrishnan
*/
public class CinaBundleDataSetHandlerTest extends AbstractFileSystemTestCase
public class CinaBundleDataSetHandlerTest extends CinaBundleRegistrationTest
{
private Mockery context;
private IEncapsulatedOpenBISService openbisService;
private CinaBundleDataSetHandler handler;
private IDataSetHandlerRpc delegator;
private ExternalData externalData;
@Override
@BeforeMethod
public void setUp() throws IOException
/**
* First set up expectations for the case that the entities do not exist, then set up the
* expectations for existing entities to simulate registering new entities.
*/
@Test
public void testHandlingWithCreationOfNewEntities()
{
super.setUp();
context = new Mockery();
openbisService = context.mock(IEncapsulatedOpenBISService.class);
setupOpenBisExpectations();
setupSessionContextExpectations();
setupCallerDataSetInfoExpectations();
delegator = context.mock(IDataSetHandlerRpc.class);
File dataSetFile =
new File("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/");
setupNewEntititesExpectations();
setupExistingGridPrepExpectations();
setupExistingReplicaExpectations();
setupHandleRawDataSetExpectations("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/RawData/ReplicTest");
setupHandleReplicaMetadataDataSetExpectations("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/Annotations/ReplicTest");
setupHandleBundleMetadataDataSetExpectations("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/BundleMetadata.xml");
initializeDataSetHandler();
}
handler.handleDataSet(dataSetFile);
@AfterMethod
public void tearDown()
{
context.assertIsSatisfied();
}
// TODO: Changing the code used by this test; once finished, I'll fix the test to match the new
// code.
@Test(groups = "broken")
public void testHandling()
/**
* Set up the expectations for existing entities to simulate registering updating entities.
*/
@Test
public void testRegistratorForExistingEntities()
{
setupOpenBisExpectations();
setupSessionContextExpectations();
setupCallerDataSetInfoExpectations();
File dataSetFile =
new File("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/");
setupDataSetHandlerExpectations(dataSetFile);
context.checking(new Expectations()
{
private long uniqueId = 1;
private void delegatorHandleDataSetExpectation(final String path)
{
final DataSetInformation dataSetInformation = new DataSetInformation();
dataSetInformation.setDataSetCode("Derived");
dataSetInformation.setSampleCode("" + uniqueId++);
dataSetInformation.setSpaceCode("Space");
dataSetInformation.setInstanceCode("Test");
allowing(delegator).linkAndHandleDataSet(with(new File(path)),
with(any(DataSetInformation.class)));
will(returnValue(Collections.singletonList(dataSetInformation)));
}
{
allowing(openbisService).drawANewUniqueID();
will(returnValue(uniqueId++));
allowing(openbisService).registerSample(with(any(NewSample.class)),
with("test"));
will(returnValue(new Long(1)));
delegatorHandleDataSetExpectation("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/Annotations/Replica for MRC files/MRC for Thomas/test20090422_BacklashRef.mrc");
delegatorHandleDataSetExpectation("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/Annotations/Replica for MRC files/MRC for Thomas/test20090424_TrackAtZeroRef.mrc");
delegatorHandleDataSetExpectation("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/Annotations/Replica for STEM files/STEM/stem_134588_1.imag");
}
});
setupExistingGridPrepExpectations();
setupExistingReplicaExpectations();
setupHandleReplicaMetadataDataSetExpectations("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/Annotations/ReplicTest");
setupHandleBundleMetadataDataSetExpectations("sourceTest/java/ch/systemsx/cisd/cina/shared/metadata/Test.bundle/BundleMetadata.xml");
initializeDataSetHandler();
handler.handleDataSet(dataSetFile);
context.assertIsSatisfied();
......@@ -131,68 +93,4 @@ public class CinaBundleDataSetHandlerTest extends AbstractFileSystemTestCase
handler = new CinaBundleDataSetHandler(props, delegator, openbisService);
}
private void setupOpenBisExpectations()
{
final SampleType sampleType = new SampleType();
sampleType.setCode(CinaConstants.REPLICA_SAMPLE_TYPE_CODE);
sampleType.setAutoGeneratedCode(true);
sampleType.setGeneratedCodePrefix("Replica-");
DataSetType dataSetType = new DataSetType(CinaConstants.IMAGE_DATA_SET_TYPE_CODE);
final DataSetTypeWithVocabularyTerms dataSetTypeWithTerms =
new DataSetTypeWithVocabularyTerms();
dataSetTypeWithTerms.setDataSetType(dataSetType);
externalData = new ExternalData();
externalData.setCode("1");
// set up the expectations
context.checking(new Expectations()
{
{
one(openbisService).getSampleType(CinaConstants.REPLICA_SAMPLE_TYPE_CODE);
will(returnValue(sampleType));
one(openbisService).getDataSetType(CinaConstants.IMAGE_DATA_SET_TYPE_CODE);
will(returnValue(dataSetTypeWithTerms));
one(openbisService).tryGetDataSet("session-token", externalData.getCode());
will(returnValue(externalData));
}
});
}
private void setupDataSetHandlerExpectations(final File dataSet)
{
final DataSetInformation dataSetInformation = new DataSetInformation();
dataSetInformation.setDataSetCode(externalData.getCode());
dataSetInformation.setSampleCode("GRID-1");
dataSetInformation.setSpaceCode("Space");
dataSetInformation.setInstanceCode("Test");
final Sample sample = new Sample();
Experiment exp = new Experiment();
exp.setIdentifier("/Space/Exp-1");
sample.setExperiment(exp);
sample.setIdentifier(dataSetInformation.getSampleIdentifier().toString());
final SessionContextDTO sessionContext = new SessionContextDTO();
sessionContext.setSessionToken("session-token");
sessionContext.setUserEmail("test@test.bar");
sessionContext.setUserName("test");
// set up the expectations
context.checking(new Expectations()
{
{
one(delegator).handleDataSet(dataSet);
will(returnValue(Collections.singletonList(dataSetInformation)));
allowing(delegator).getSessionContext();
will(returnValue(sessionContext));
one(delegator).getFileForExternalData(externalData);
will(returnValue(dataSet.getParentFile()));
allowing(openbisService).tryGetSampleWithExperiment(
dataSetInformation.getSampleIdentifier());
will(returnValue(sample));
}
});
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment