diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/impl/FeatureVectorContainerDataSet.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/impl/FeatureVectorContainerDataSet.java
index 3047aa608fef028eb0ed5b30a40cfa5665b83cfa..57fefcd6d3d5b2e2230f4fdb334736a5c735c083 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/impl/FeatureVectorContainerDataSet.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/impl/FeatureVectorContainerDataSet.java
@@ -98,12 +98,6 @@ public class FeatureVectorContainerDataSet extends DataSet<DataSetInformation> i
getRegistrationDetails().setPropertyValue(propertyCode, propertyValue);
}
- @Override
- public File getDataSetStagingFolder()
- {
- return originalDataset.getDataSetStagingFolder();
- }
-
public static String getContainerAnalysisType(String dataSetTypeCode)
{
if (!isHCSAnalysisDataSetType(dataSetTypeCode))
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/impl/FeatureVectorDataSet.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/impl/FeatureVectorDataSet.java
index 25d2ccaa59754f8eac1a0361d9c7257cd1098375..8f10f1878bcd079c2c5e6fccf89527b6f8f1ffaa 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/impl/FeatureVectorDataSet.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v2/impl/FeatureVectorDataSet.java
@@ -64,12 +64,6 @@ public class FeatureVectorDataSet extends DataSet<FeatureVectorDataSetInformatio
return dataSet.getRegistrationDetails();
}
- @Override
- public File getDataSetStagingFolder()
- {
- return dataSet.getDataSetStagingFolder();
- }
-
@Override
public String getDataSetCode()
{
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/screening/server/DssServiceRpcScreening.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/screening/server/DssServiceRpcScreening.java
index d1ed76c9a1cfc3c7b65210791a31dbf1ed6a1a41..b925397281eca80160075a5b0e52f2d4c7c70c4d 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/screening/server/DssServiceRpcScreening.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/screening/server/DssServiceRpcScreening.java
@@ -1319,14 +1319,12 @@ public class DssServiceRpcScreening extends AbstractDssServiceRpc<IDssServiceRpc
{
String[] permIds = extractPermIds(datasetIdents);
List<ImgAnalysisDatasetDTO> datasets = getDAO().listAnalysisDatasetsByPermId(permIds);
- if (datasets.size() != datasetIdents.size())
+
+ if (datasets.size() == 0 && datasetIdents.size() > 0)
{
- Set<String> missing = new HashSet<String>(Arrays.asList(permIds));
- for (ImgAnalysisDatasetDTO dataset : datasets)
- {
- missing.remove(dataset.getPermId());
- }
- throw new UserFailureException("Following datasets could not be found: " + missing);
+ throw new UserFailureException(
+ "Couldn't find any analysis dataset for given datasets: "
+ + Arrays.asList(permIds));
}
return datasets;
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java
index 8179526ba36c85333e2ef2d1ebcc72c0d0820cb3..b3d853b699e625620712535cdce60fc82a9f73a2 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/plugin/screening/client/api/v1/ScreeningOpenbisServiceFacade.java
@@ -13,6 +13,7 @@ import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
+import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
@@ -23,12 +24,12 @@ import org.apache.commons.lang.time.DateUtils;
import ch.systemsx.cisd.base.image.IImageTransformerFactory;
import ch.systemsx.cisd.common.api.MinimalMinorVersion;
-import ch.systemsx.cisd.openbis.common.api.client.ServiceFinder;
import ch.systemsx.cisd.common.api.retry.RetryCaller;
import ch.systemsx.cisd.common.api.retry.RetryProxyFactory;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.io.ConcatenatedFileOutputStreamWriter;
+import ch.systemsx.cisd.openbis.common.api.client.ServiceFinder;
import ch.systemsx.cisd.openbis.dss.client.api.v1.DssComponentFactory;
import ch.systemsx.cisd.openbis.dss.client.api.v1.IDataSetDss;
import ch.systemsx.cisd.openbis.dss.client.api.v1.IDssComponent;
@@ -51,6 +52,7 @@ import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.Sample;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria.MatchClause;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria.MatchClauseAttribute;
+import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchCriteria.SearchOperator;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchSubCriteria;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.filter.IDataSetFilter;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.filter.TypeBasedDataSetFilter;
@@ -200,8 +202,8 @@ public class ScreeningOpenbisServiceFacade implements IScreeningOpenbisServiceFa
/**
* This constructor is only for use in tests. Do not use it otherwise.
*/
- public static IScreeningOpenbisServiceFacade tryCreateForTest(String sessionToken, String serverUrl,
- final IScreeningApiServer openbisServer)
+ public static IScreeningOpenbisServiceFacade tryCreateForTest(String sessionToken,
+ String serverUrl, final IScreeningApiServer openbisServer)
{
return tryCreate(sessionToken, serverUrl, openbisServer);
}
@@ -902,7 +904,30 @@ public class ScreeningOpenbisServiceFacade implements IScreeningOpenbisServiceFa
List<? extends IFeatureVectorDatasetIdentifier> featureDatasets)
{
final Set<FeatureInformation> result = new HashSet<FeatureInformation>();
- featureVectorDataSetIdentifierMultiplexer.process(featureDatasets,
+
+ List<DataSet> containedDataSets = getContainedDataSets(featureDatasets);
+
+ List<IFeatureVectorDatasetIdentifier> identifiersIncludingContained = null;
+
+ if (containedDataSets.size() != 0)
+ {
+ String host = featureDatasets.get(0).getDatastoreServerUrl();
+
+ identifiersIncludingContained = new LinkedList<IFeatureVectorDatasetIdentifier>();
+ identifiersIncludingContained.addAll(featureDatasets);
+ for (DataSet contained : containedDataSets)
+ {
+ IFeatureVectorDatasetIdentifier containedIdentifier =
+ new FeatureVectorDatasetReference(contained.getCode(),
+ contained.getDataSetTypeCode(), host, null, null, null, null, null,
+ null);
+ identifiersIncludingContained.add(containedIdentifier);
+ }
+ }
+
+ featureVectorDataSetIdentifierMultiplexer.process(
+ identifiersIncludingContained == null ? featureDatasets
+ : identifiersIncludingContained,
new IReferenceHandler<IFeatureVectorDatasetIdentifier>()
{
@Override
@@ -932,6 +957,30 @@ public class ScreeningOpenbisServiceFacade implements IScreeningOpenbisServiceFa
return new ArrayList<FeatureInformation>(result);
}
+ private SearchSubCriteria getContainerSearchCriteria(String containerCode)
+ {
+ SearchCriteria searchSubCriteria = new SearchCriteria();
+ searchSubCriteria.addMatchClause(MatchClause.createAttributeMatch(
+ MatchClauseAttribute.CODE, containerCode));
+ return SearchSubCriteria.createDataSetContainerCriteria(searchSubCriteria);
+ }
+
+ protected List<DataSet> getContainedDataSets(
+ List<? extends IFeatureVectorDatasetIdentifier> featureDatasets)
+ {
+ SearchCriteria searchCriteria = new SearchCriteria();
+ for (IFeatureVectorDatasetIdentifier id : featureDatasets)
+ {
+ String containerId = id.getDatasetCode();
+ searchCriteria.addSubCriteria(getContainerSearchCriteria(containerId));
+ }
+ searchCriteria.setOperator(SearchOperator.MATCH_ANY_CLAUSES);
+
+ List<DataSet> containedDataSets =
+ generalInformationService.searchForDataSets(sessionToken, searchCriteria);
+ return containedDataSets;
+ }
+
/**
* For a given set of plates and a set of features (given by their code), provide all the
* feature vectors.
diff --git a/screening/sourceTest/core-plugins/FeatureVectorsDropboxTest/1/dss/drop-boxes/FeatureVectorsDropboxTest-drop-box/plugin.properties b/screening/sourceTest/core-plugins/FeatureVectorsDropboxTest/1/dss/drop-boxes/FeatureVectorsDropboxTest-drop-box/plugin.properties
index 900b82a6dbb9b13c2d529a9eb13bb82341382164..8ab36658292a7ae31514007026cb39b1d8176a7a 100644
--- a/screening/sourceTest/core-plugins/FeatureVectorsDropboxTest/1/dss/drop-boxes/FeatureVectorsDropboxTest-drop-box/plugin.properties
+++ b/screening/sourceTest/core-plugins/FeatureVectorsDropboxTest/1/dss/drop-boxes/FeatureVectorsDropboxTest-drop-box/plugin.properties
@@ -3,6 +3,16 @@ incoming-dir-create = true
incoming-data-completeness-condition = auto-detection
top-level-data-set-handler = ch.systemsx.cisd.openbis.dss.etl.jython.v2.JythonPlateDataSetHandlerV2
script-path = feature-vector-data-set-handler.py
-storage-processor = ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor
-storage-processor.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
-storage-processor.data-source = imaging-db
\ No newline at end of file
+storage-processor = ch.systemsx.cisd.etlserver.DispatcherStorageProcessor
+storage-processor.processors = images, analysis
+# --- images - accepts all image datasets registrations
+storage-processor.images = ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor
+storage-processor.images.data-source = imaging-db
+# --- analysis - accepts all other datasets (and assumes they are single CSV files)
+storage-processor.analysis = ch.systemsx.cisd.openbis.dss.etl.featurevector.FeatureVectorStorageProcessor
+storage-processor.analysis.processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
+storage-processor.analysis.data-source = imaging-db
+storage-processor.analysis.separator = ,
+storage-processor.analysis.well-name-row = Well
+storage-processor.analysis.well-name-col = Well
+storage-processor.analysis.well-name-col-is-alphanum = true
diff --git a/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/FeatureVectorsDropboxTest.java b/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/FeatureVectorsDropboxTest.java
index 63ca230492536ebd45535ab3a2518fa9bbde90b6..2165c506dd0a04d2fae92863dd899e24c9218dcd 100644
--- a/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/FeatureVectorsDropboxTest.java
+++ b/screening/sourceTest/java/ch/systemsx/cisd/openbis/screening/systemtests/FeatureVectorsDropboxTest.java
@@ -19,6 +19,8 @@ package ch.systemsx.cisd.openbis.screening.systemtests;
import java.io.File;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.LinkedList;
import java.util.List;
import org.apache.commons.io.FileUtils;
@@ -30,11 +32,15 @@ import org.testng.annotations.Test;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.servlet.SpringRequestContextProvider;
+import ch.systemsx.cisd.openbis.generic.server.util.TestInitializer;
import ch.systemsx.cisd.openbis.plugin.screening.client.api.v1.IScreeningOpenbisServiceFacade;
import ch.systemsx.cisd.openbis.plugin.screening.client.api.v1.ScreeningOpenbisServiceFacade;
import ch.systemsx.cisd.openbis.plugin.screening.client.web.client.IScreeningClientService;
import ch.systemsx.cisd.openbis.plugin.screening.shared.ResourceNames;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.IScreeningApiServer;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureInformation;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVector;
+import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVectorDataset;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVectorDatasetReference;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Plate;
@@ -62,7 +68,22 @@ public class FeatureVectorsDropboxTest extends AbstractScreeningSystemTestCase
moveFileToIncoming(exampleDataSet);
waitUntilDataSetImported();
}
-
+
+ @Override
+ protected boolean checkLogContentForFinishedDataSetRegistration(String logContent)
+ {
+ return checkOnFinishedPostRegistration(logContent);
+ }
+
+ /**
+ * sets up the openbis database to be used by the tests.
+ */
+ @Override
+ protected void setUpDatabaseProperties()
+ {
+ TestInitializer.initEmptyDbWithIndex();
+ }
+
@BeforeMethod
public void setUp() throws Exception
{
@@ -75,7 +96,9 @@ public class FeatureVectorsDropboxTest extends AbstractScreeningSystemTestCase
Object bean = applicationContext.getBean(ResourceNames.SCREENING_PLUGIN_SERVER);
screeningServer = (IScreeningApiServer) bean;
sessionToken = screeningClientService.tryToLogin("admin", "a").getSessionID();
- screeningFacade = ScreeningOpenbisServiceFacade.tryCreateForTest(sessionToken, "http://localhost:" + SYSTEM_TEST_CASE_SERVER_PORT, screeningServer);
+ screeningFacade =
+ ScreeningOpenbisServiceFacade.tryCreateForTest(sessionToken, "http://localhost:"
+ + SYSTEM_TEST_CASE_SERVER_PORT, screeningServer);
}
@AfterMethod
@@ -102,8 +125,40 @@ public class FeatureVectorsDropboxTest extends AbstractScreeningSystemTestCase
FeatureVectorDatasetReference feature = features.get(0);
assertEquals("HCS_ANALYSIS_CONTAINER_WELL_FEATURES", feature.getDataSetType());
+ List<FeatureInformation> availableFeatures =
+ screeningFacade.listAvailableFeatures(features);
+
+ List<String> availableFeatureCodes = new LinkedList<String>();
+ for (FeatureInformation availableFeature : availableFeatures)
+ {
+ availableFeatureCodes.add(availableFeature.getCode());
+ }
+ Collections.sort(availableFeatureCodes);
+ assertEquals("[COLUMN_NUMBER, ROW_NUMBER, STATE, TPU]", availableFeatureCodes.toString());
+
+ List<FeatureVectorDataset> featureVectorDatasets =
+ screeningFacade.loadFeatures(features, Collections.singletonList("TPU"));
+
+ assertEquals(1, featureVectorDatasets.size());
+
+ FeatureVectorDataset featureVectorDataset = featureVectorDatasets.get(0);
+
+ int nonNanPositions = 0;
+
+ for (FeatureVector v : featureVectorDataset.getFeatureVectors())
+ {
+ if (false == Double.isNaN(v.getValues()[0]))
+ {
+ nonNanPositions++;
+ }
+ }
+
+ assertEquals(
+ "There are not enough values in the feature. Maybe the feature was not loaded correctly",
+ 8 * 12, nonNanPositions);
+
}
-
+
private List<Plate> filterPlates(List<Plate> plates)
{
List<Plate> filteredPlates = new ArrayList<Plate>();
@@ -125,6 +180,7 @@ public class FeatureVectorsDropboxTest extends AbstractScreeningSystemTestCase
File featureSrc = getFeatureVectorsTestData();
FileUtils.copyFileToDirectory(featureSrc, dest);
// Copy the test data set to the location for processing
+
return dest;
}