diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/ExperimentToDataSetRelatedEntitiesTranslator.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/ExperimentToDataSetRelatedEntitiesTranslator.java new file mode 100644 index 0000000000000000000000000000000000000000..dfd2066877cbf394a73d8d22847c559b989481a8 --- /dev/null +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/ExperimentToDataSetRelatedEntitiesTranslator.java @@ -0,0 +1,87 @@ +package ch.systemsx.cisd.openbis.generic.server.api.v1; + +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.Experiment; +import ch.systemsx.cisd.openbis.generic.shared.basic.BasicEntityInformationHolder; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetRelatedEntities; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityKind; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityType; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExperimentType; + +/** + * A class that converts {@link Experiment} objects to {@link DataSetRelatedEntities} objects. + * + * @author Chandrasekhar Ramakrishnan + */ +class ExperimentToDataSetRelatedEntitiesTranslator +{ + // A map from experiment type Id to experiment type. + private final HashMap<String, ExperimentType> experimentTypesMap; + + private final List<Experiment> experiments; + + private final ArrayList<BasicEntityInformationHolder> entityInformationHolders; + + /** + * Creates a translator from public {@Experiment} objects to the internal + * {@link DataSetRelatedEntities} objects. + * <p> + * A list of experiment types known to the DB must be provided because Experiment knows only the + * code of the ExperimentType. + * + * @param experimentTypes A list of ExperimentTypes known to the DB. + * @param experiments The experiments to convert. + */ + public ExperimentToDataSetRelatedEntitiesTranslator(List<ExperimentType> experimentTypes, + List<Experiment> experiments) + { + this.experimentTypesMap = convertExperimentTypesListToMap(experimentTypes); + this.experiments = experiments; + entityInformationHolders = new ArrayList<BasicEntityInformationHolder>(experiments.size()); + } + + private static HashMap<String, ExperimentType> convertExperimentTypesListToMap( + List<ExperimentType> experimentTypes) + { + HashMap<String, ExperimentType> map = + new HashMap<String, ExperimentType>(experimentTypes.size()); + + for (ExperimentType experimentType : experimentTypes) + { + map.put(experimentType.getCode(), experimentType); + } + + return map; + } + + public DataSetRelatedEntities convertToDataSetRelatedEntities() + { + for (Experiment experiment : experiments) + { + BasicEntityInformationHolder holderOrNull = + tryConvertExperimentToEntityInformationHolder(experiment); + if (null != holderOrNull) + { + entityInformationHolders.add(holderOrNull); + } + } + return new DataSetRelatedEntities(entityInformationHolders); + } + + private BasicEntityInformationHolder tryConvertExperimentToEntityInformationHolder( + Experiment experiment) + { + EntityType entityType = experimentTypesMap.get(experiment.getExperimentTypeCode()); + if (null == entityType) + { + return null; + } + BasicEntityInformationHolder holder = + new BasicEntityInformationHolder(EntityKind.EXPERIMENT, entityType, + experiment.getCode(), experiment.getId(), experiment.getPermId()); + return holder; + } +} \ No newline at end of file diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/GeneralInformationService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/GeneralInformationService.java index 7b61f83961ba57e5a6f8cc68a621efbf19fa3b0c..75736ae08149ac6002fe9562c8f29d13114eb65f 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/GeneralInformationService.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/GeneralInformationService.java @@ -165,7 +165,7 @@ public class GeneralInformationService extends AbstractServer<IGeneralInformatio public int getMinorVersion() { - return 13; + return 14; } private Map<String, List<RoleAssignmentPE>> getRoleAssignmentsPerSpace() @@ -374,7 +374,8 @@ public class GeneralInformationService extends AbstractServer<IGeneralInformatio commonServer.listVocabularies(sessionToken, true, false); for (ch.systemsx.cisd.openbis.generic.shared.basic.dto.Vocabulary privateVocabulary : privateVocabularies) { - vocabTerms.put(privateVocabulary, Translator.translatePropertyTypeTerms(privateVocabulary.getTerms())); + vocabTerms.put(privateVocabulary, + Translator.translatePropertyTypeTerms(privateVocabulary.getTerms())); } return vocabTerms; } @@ -407,6 +408,22 @@ public class GeneralInformationService extends AbstractServer<IGeneralInformatio return Translator.translate(dataSets, connectionsToGet); } + public List<DataSet> listDataSetsForExperiments(String sessionToken, + List<Experiment> experiments, EnumSet<Connections> connections) + { + checkSession(sessionToken); + EnumSet<Connections> connectionsToGet = + (connections != null) ? connections : EnumSet.noneOf(Connections.class); + + List<ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExperimentType> experimentTypes = + commonServer.listExperimentTypes(sessionToken); + ExperimentToDataSetRelatedEntitiesTranslator translator = + new ExperimentToDataSetRelatedEntitiesTranslator(experimentTypes, experiments); + DataSetRelatedEntities dsre = translator.convertToDataSetRelatedEntities(); + List<ExternalData> dataSets = commonServer.listRelatedDataSets(sessionToken, dsre); + return Translator.translate(dataSets, connectionsToGet); + } + public List<DataSet> getDataSetMetaData(String sessionToken, List<String> dataSetCodes) { Session session = getSession(sessionToken); diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/GeneralInformationServiceLogger.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/GeneralInformationServiceLogger.java index f3b3a2ec1ba4d790104a432281a1bf6c720bd6b2..03be8528e7ce75be0853d86649dd118b70e6c63c 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/GeneralInformationServiceLogger.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/api/v1/GeneralInformationServiceLogger.java @@ -174,4 +174,12 @@ class GeneralInformationServiceLogger extends AbstractServerLogger implements logAccess(sessionToken, "list-vocabularies"); return null; } + + public List<DataSet> listDataSetsForExperiments(String sessionToken, + List<Experiment> experiments, EnumSet<Connections> connectionsToGet) + { + logAccess(sessionToken, "list-data-sets-for-experiments", + "EXPERIMENTS(%s) CONNECTIONS(%s)", experiments, connectionsToGet); + return null; + } } diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/api/v1/IGeneralInformationService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/api/v1/IGeneralInformationService.java index 9b6041864a684af059cbcabd841e3afb1673a56f..ca8fe924231d8e962bb1bb128b23c521b3319d54 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/api/v1/IGeneralInformationService.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/api/v1/IGeneralInformationService.java @@ -222,6 +222,18 @@ public interface IGeneralInformationService extends IRpcService public List<DataSet> listDataSets(String sessionToken, List<Sample> samples, EnumSet<Connections> connectionsToGet); + /** + * Return all data sets attached to the given experiments with connections. Available since + * minor version 14. + * + * @param experiments The experiments for which we return attached data sets. + * @since 1.14 + */ + @Transactional(readOnly = true) + @RolesAllowed(RoleWithHierarchy.INSTANCE_OBSERVER) + public List<DataSet> listDataSetsForExperiments(String sessionToken, + List<Experiment> experiments, EnumSet<Connections> connectionsToGet); + /** * Returns meta data for all specified data sets. This contains data set type, properties, and * codes of linked parent and children data sets. Available since minor version 12. diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/remoteapitest/api/v1/GeneralInformationServiceJsonApiTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/remoteapitest/api/v1/GeneralInformationServiceJsonApiTest.java index 3fe9c53becf3462a21a4152ed5bca2526e6167a2..3a364e113496a603ae584c733300f917c0977211 100644 --- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/remoteapitest/api/v1/GeneralInformationServiceJsonApiTest.java +++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/remoteapitest/api/v1/GeneralInformationServiceJsonApiTest.java @@ -23,6 +23,7 @@ import static org.testng.AssertJUnit.fail; import java.net.MalformedURLException; import java.util.ArrayList; +import java.util.Arrays; import java.util.Collections; import java.util.Comparator; import java.util.EnumSet; @@ -283,18 +284,31 @@ public class GeneralInformationServiceJsonApiTest extends RemoteApiTestCase generalInformationService.listDataSets(sessionToken, samples, EnumSet.of(Connections.PARENTS)); assertEquals(true, result.size() > 0); - // See if some sample has parents - boolean parentCodesFound = false; for (DataSet dataSet : result) { - if (false == dataSet.getParentCodes().isEmpty()) - { - parentCodesFound = true; - break; - } + assertEquals( + "No parent codes should have been found for data set " + dataSet.getCode(), + "[]", dataSet.getParentCodes().toString()); + } + } + + @Test + public void testListDataSetsWithParentsForAllExperiments() + { + List<String> experimentIdentifiers = Arrays.asList("/CISD/DEFAULT/EXP-REUSE"); + List<Experiment> experiments = + generalInformationService.listExperiments(sessionToken, experimentIdentifiers); + List<DataSet> result = + generalInformationService.listDataSetsForExperiments(sessionToken, experiments, + EnumSet.of(Connections.PARENTS)); + assertEquals(true, result.size() > 0); + for (DataSet dataSet : result) + { + assertEquals( + "No parent codes should have been found for data set " + dataSet.getCode(), + "[]", dataSet.getParentCodes().toString()); } - assertTrue("No parent codes should have been found", (false == parentCodesFound)); } @Test diff --git a/screening/source/java/OpenBISScreeningML.java b/screening/source/java/OpenBISScreeningML.java index f12abaacb761df7f601874de8f07965984d77f99..310d36e455936ff8c13024ba29f6d3ab5b3efea7 100644 --- a/screening/source/java/OpenBISScreeningML.java +++ b/screening/source/java/OpenBISScreeningML.java @@ -858,31 +858,9 @@ public class OpenBISScreeningML openbis.getFullDataSets(plateIdentifier, new AndDataSetFilter( new TypeBasedDataSetFilter(dataSetTypeCodePattern), new PropertiesBasedDataSetFilter(createMap(properties)))); - Object[][] result = new Object[dataSets.size()][]; try { - for (int i = 0; i < dataSets.size(); i++) - { - DataSet dataSet = dataSets.get(i); - String code = dataSet.getCode(); - File file = new File(dataSetsDir, code); - if (file.exists() == false) - { - file = - dataSet.getLinkOrCopyOfContents(overrideStoreRootPathOrNull, - dataSetsDir); - } - List<String> parents = dataSet.getParentCodes(); - Object[] parentCodes = new Object[parents.size()]; - for (int j = 0; j < parentCodes.length; j++) - { - parentCodes[j] = parents.get(j); - } - Object[][] dataSetProperties = listProperties(dataSet.getProperties()); - result[i] = new Object[] - { code, file.getPath(), dataSetProperties, parentCodes }; - } - return result; + return translateDataSets(overrideStoreRootPathOrNull, dataSets); } catch (Exception ex) { throw createException("Loading data sets for plate '" + augmentedPlateCode @@ -890,6 +868,95 @@ public class OpenBISScreeningML } } + /** + * Loads data sets for specified experiment code. For each data set the path to the root of the + * data set is returned. If it is possible the path points directly into the data set store. No + * data is copied. Otherwise the data is retrieved from the data store server.<br> + * If the same dataset is loaded for the second time in one session it will be immediately + * returned from the local cache. + * <p> + * Matlab example: + * + * <pre> + * % Load all data sets of plate P005 in space SPACE + * properties = {'ANALYSIS_PROCEDURE' 'AX87'} + * dsinfo = OpenBISScreeningML.loadDataSets('/SPACE/PROJECT/EXPERIMENT', 'HCS_ANALYSIS_CELL_FEATURES_CC_MAT', properties, '/mount/openbis-store') + * % Get the data set codes + * dsinfo(:,1) + * % Get root path of first data set (assuming there is at least one) + * dsinfo(1,2) + * % Get the properties for the first data set + * props = dsinfo(1,3) + * % Get property key of first property + * props(1,1) + * % Get property value of first property + * props(1,2) + * % Get all parents of first data set (assuming there is at least one) + * dsInfo(1,4) + * </pre> + * + * @param augmentedExperimentCode The augmented experiment code. + * @param dataSetTypeCodePattern only data sets of the type which matches the specified pattern + * will be returned. To fetch all data sets specify ".*". + * @param properties Only data set with specified property values will be returned. This is a + * two dimensional array where the first column contains the property codes and the + * second column the corresponding property values. + * @return Each row contains information about one data set: + * @param overrideStoreRootPathOrNull A path, in the context of the local file system mounts, to + * the DSS' store root. If null, paths are returned in the context of the DSS' file + * system mounts. + * <p> + * <code>{ data set code, data set root path, { {key1, value1}, {key2, value2} ...}, parents }</code> + */ + public static Object[][] loadDataSetsForExperiment(String augmentedExperimentCode, + final String dataSetTypeCodePattern, final Object[][] properties, + String overrideStoreRootPathOrNull) + { + checkLoggedIn(); + ExperimentIdentifier experimentIdentifier = + getExperimentIdentifierOrFail(augmentedExperimentCode); + List<DataSet> dataSets = + openbis.getFullDataSets(experimentIdentifier, new AndDataSetFilter( + new TypeBasedDataSetFilter(dataSetTypeCodePattern), + new PropertiesBasedDataSetFilter(createMap(properties)))); + try + { + return translateDataSets(overrideStoreRootPathOrNull, dataSets); + } catch (Exception ex) + { + throw createException("Loading data sets for experiment '" + augmentedExperimentCode + + "' failed.", ex); + } + } + + private static Object[][] translateDataSets(String overrideStoreRootPathOrNull, + List<DataSet> dataSets) + { + Object[][] result = new Object[dataSets.size()][]; + for (int i = 0; i < dataSets.size(); i++) + { + DataSet dataSet = dataSets.get(i); + String code = dataSet.getCode(); + File file = new File(dataSetsDir, code); + if (file.exists() == false) + { + file = + dataSet.getLinkOrCopyOfContents(overrideStoreRootPathOrNull, + dataSetsDir); + } + List<String> parents = dataSet.getParentCodes(); + Object[] parentCodes = new Object[parents.size()]; + for (int j = 0; j < parentCodes.length; j++) + { + parentCodes[j] = parents.get(j); + } + Object[][] dataSetProperties = listProperties(dataSet.getProperties()); + result[i] = new Object[] + { code, file.getPath(), dataSetProperties, parentCodes }; + } + return result; + } + /** * Loads file/folder of specified data set and specified file/folder path inside the data set. * If it is possible the path points directly into the data set store. No data is copied. diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/IScreeningOpenbisServiceFacade.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/IScreeningOpenbisServiceFacade.java index d61a37fb79c67187a7ca300d8a86e03be7dd798c..919570c88b6f47e31e47b7e254e94c53b52d768f 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/IScreeningOpenbisServiceFacade.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/IScreeningOpenbisServiceFacade.java @@ -281,6 +281,19 @@ public interface IScreeningOpenbisServiceFacade public List<DataSet> getFullDataSets(PlateIdentifier plateIdentifier, IDataSetFilter dataSetFilter) throws IllegalStateException, EnvironmentFailureException; + /** + * A list of data sets owned by specified experiment and passing specified filter. The data set + * objects provide metadata (e.g. code, properties etc. from the openBIS AS) as well as data + * (e.g. files from openBIS DSS). + * + * @throws IllegalStateException Thrown if the user has not yet been authenticated. + * @throws EnvironmentFailureException Thrown in cases where it is not possible to connect to + * the server. + */ + public List<ch.systemsx.cisd.openbis.dss.client.api.v1.DataSet> getFullDataSets( + ExperimentIdentifier experimentIdentifier, IDataSetFilter dataSetFilter) + throws IllegalStateException, EnvironmentFailureException; + /** * Returns meta data for all specified data set codes. This contains data set type, properties, * and codes of linked parent and children data sets. @@ -765,4 +778,5 @@ public interface IScreeningOpenbisServiceFacade */ public List<String> listAnalysisProcedures(ExperimentIdentifier experimentIdentifier); + } \ No newline at end of file diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java index ee80e1fe59feca1686ac7510fc2fe60eba6e1fa2..534fba670f0e3d7039da8e627d2b22f9c84447f5 100644 --- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java +++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java @@ -25,6 +25,7 @@ import ch.systemsx.cisd.base.image.IImageTransformerFactory; import ch.systemsx.cisd.common.api.MinimalMinorVersion; import ch.systemsx.cisd.common.api.client.ServiceFinder; import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; +import ch.systemsx.cisd.common.exceptions.UserFailureException; import ch.systemsx.cisd.common.io.ConcatenatedFileOutputStreamWriter; import ch.systemsx.cisd.openbis.dss.client.api.v1.DssComponentFactory; import ch.systemsx.cisd.openbis.dss.client.api.v1.IDataSetDss; @@ -43,6 +44,7 @@ import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationChangin import ch.systemsx.cisd.openbis.generic.shared.api.v1.IGeneralInformationService; import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.DataSet; import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.DataSet.Connections; +import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.Experiment; import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.Sample; import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria; import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria.MatchClause; @@ -598,6 +600,43 @@ public class ScreeningOpenbisServiceFacade implements IScreeningOpenbisServiceFa return result; } + public List<ch.systemsx.cisd.openbis.dss.client.api.v1.DataSet> getFullDataSets( + ExperimentIdentifier experimentIdentifier, IDataSetFilter dataSetFilter) + throws IllegalStateException, EnvironmentFailureException + { + List<Experiment> experiments = + generalInformationService.listExperiments(sessionToken, + Collections.singletonList(experimentIdentifier.getAugmentedCode())); + if (experiments.isEmpty()) + { + throw UserFailureException.fromTemplate("Experiment '%s' does not exist.", + experimentIdentifier); + } + return getFullDataSets(experiments, dataSetFilter); + } + + private List<ch.systemsx.cisd.openbis.dss.client.api.v1.DataSet> getFullDataSets( + final List<Experiment> experiments, IDataSetFilter filter) + { + final List<DataSet> dataSets = + generalInformationService.listDataSetsForExperiments(sessionToken, experiments, + EnumSet.of(Connections.PARENTS)); + final List<ch.systemsx.cisd.openbis.dss.client.api.v1.DataSet> result = + new ArrayList<ch.systemsx.cisd.openbis.dss.client.api.v1.DataSet>(); + for (DataSet dataSet : dataSets) + { + if (filter.pass(dataSet)) + { + ch.systemsx.cisd.openbis.dss.client.api.v1.DataSet fullDataset = + new ch.systemsx.cisd.openbis.dss.client.api.v1.DataSet( + openbisServiceFacade, dssComponent, dataSet, null); + + result.add(fullDataset); + } + } + return result; + } + public List<ch.systemsx.cisd.openbis.dss.client.api.v1.DataSet> getDataSetMetaData( List<String> dataSetCodes) { diff --git a/screening/sourceTest/java/OpenBISScreeningMLTest.java b/screening/sourceTest/java/OpenBISScreeningMLTest.java index cfac6ec1d2c53aa170c968e77d6239065c6821a6..cc873a5cb3772ba4f6f264a1b4979cca450c5ba3 100644 --- a/screening/sourceTest/java/OpenBISScreeningMLTest.java +++ b/screening/sourceTest/java/OpenBISScreeningMLTest.java @@ -726,6 +726,51 @@ public class OpenBISScreeningMLTest extends AbstractFileSystemTestCase context.assertIsSatisfied(); } + @Test + public void testLoadDataSetsForExperiment() + { + final File dataSetFolder = + new File(OpenBISScreeningML.tempDir, OpenBISScreeningML.DATASETS_FOLDER); + final File ds1Folder = new File(dataSetFolder, "ds-1"); + File ds2Folder = new File(dataSetFolder, "ds-2"); + ds2Folder.mkdirs(); + final String datasetTypePattern = "blablaCode"; + final String mountPoint = "/mount/openbis/store"; + final RecordingMatcher<IDataSetFilter> filterMatcher = + new RecordingMatcher<IDataSetFilter>(); + + List<String> codes = new ArrayList<String>(); + final DataSet dataSet1 = createDataSet("ds-1", ds1, codes, codes); + final DataSet dataSet2 = createDataSet("ds-2", ds2, codes, codes); + + context.checking(new Expectations() + { + { + one(openbis).getFullDataSets(with(eId1), with(filterMatcher)); + will(returnValue(Arrays.asList(dataSet1, dataSet2))); + + one(ds1).getLinkOrCopyOfContents(mountPoint, dataSetFolder); + will(returnValue(ds1Folder)); + + } + }); + + Object[][] result = + OpenBISScreeningML.loadDataSetsForExperiment("/S/P/E1", datasetTypePattern, + new Object[0][], + mountPoint); + + assertEquals("Type:blablaCode AND Properties:[]", filterMatcher.recordedObject().toString()); + assertEquals("ds-1", result[0][0]); + assertEquals(ds1Folder.getPath(), result[0][1]); + assertEqualProperties(dataSet1.getProperties(), (Object[][]) result[0][2]); + assertEquals("ds-2", result[1][0]); + assertEquals(ds2Folder.getPath(), result[1][1]); + assertEqualProperties(dataSet2.getProperties(), (Object[][]) result[1][2]); + assertEquals(2, result.length); + context.assertIsSatisfied(); + } + @Test public void testUpdateDataSet() {