diff --git a/datastore_server/etc/service.properties b/datastore_server/etc/service.properties
index 21d6021e6f979dcdab9110ac5f4fab9771912758..6da5a9c9691228400a045aab28bf866a60731178 100644
--- a/datastore_server/etc/service.properties
+++ b/datastore_server/etc/service.properties
@@ -212,6 +212,8 @@ validator.site.value-range = [0,Infinity)
# E.g. 'code-extractor' property for the thread 'my-etl' should be specified as 'my-etl.code-extractor'
inputs=main-thread, tsv-thread, csv-thread, simple-thread, hdf5-thread, dss-system-test-thread
+dss-rpc.put.CUSTOM-IMPORT = test-custom-import
+
# True if incoming directories should be created on server startup if they don't exist.
# Default - false (server will fail at startup if one of incoming directories doesn't exist).
incoming-dir-create = true
diff --git a/datastore_server/source/core-plugins/core-plugins.properties b/datastore_server/source/core-plugins/core-plugins.properties
index ad076de30cf2a46969625829ac64e57daf8d3a76..36cf1a47b4385a3b705c054d2d457a8e7f259776 100644
--- a/datastore_server/source/core-plugins/core-plugins.properties
+++ b/datastore_server/source/core-plugins/core-plugins.properties
@@ -6,7 +6,7 @@
# List of comma-separated regular expressions. If a technology (i.e. module) from the core plugins
# folder is matching one of these regular expressions it is enabled.
#
-#enabled-modules =
+enabled-modules = import-test
#
# List of comma-separated full core plugin names (or prefixes) of core plugins to be disabled
diff --git a/datastore_server/source/core-plugins/import-test/1/as/custom-imports/test-custom-import/plugin.properties b/datastore_server/source/core-plugins/import-test/1/as/custom-imports/test-custom-import/plugin.properties
new file mode 100644
index 0000000000000000000000000000000000000000..fcbdb19ef13cab01d292b9a91fddadf0fea36323
--- /dev/null
+++ b/datastore_server/source/core-plugins/import-test/1/as/custom-imports/test-custom-import/plugin.properties
@@ -0,0 +1,4 @@
+name = Test custom import
+dss-code = STANDARD
+dropbox-name = test-custom-import
+description = This is a test custom import
\ No newline at end of file
diff --git a/datastore_server/source/core-plugins/import-test/1/as/services/import-test/import-test.py b/datastore_server/source/core-plugins/import-test/1/as/services/import-test/import-test.py
new file mode 100644
index 0000000000000000000000000000000000000000..ddf1d71e354693600f1cf287d2cfa83379490ae2
--- /dev/null
+++ b/datastore_server/source/core-plugins/import-test/1/as/services/import-test/import-test.py
@@ -0,0 +1,36 @@
+def process(context, parameters):
+ print(">>> import-test <<<");
+
+ sessionToken = context.getSessionToken()
+ operation = parameters.get("operation");
+ uploadKey = parameters.get("uploadKey");
+ typeCode = parameters.get("typeCode");
+ asynchrounous = parameters.get("async");
+ userEmail = parameters.get("userEmail");
+ defaultSpaceIdentifier = parameters.get("defaultSpaceIdentifier");
+ spaceIdentifierOverride = parameters.get("spaceIdentifierOverride");
+ experimentIdentifierOverride = parameters.get("experimentIdentifierOverride");
+ updateExisting = parameters.get("updateExisting");
+ ignoreUnregistered = parameters.get("ignoreUnregistered");
+ customImportCode = parameters.get("customImportCode");
+
+ if operation == "createExperiments":
+ return context.getImportService().createExperiments(sessionToken, uploadKey, typeCode, asynchrounous, userEmail);
+ elif operation == "updateExperiments":
+ return context.getImportService().updateExperiments(sessionToken, uploadKey, typeCode, asynchrounous, userEmail);
+ elif operation == "createSamples":
+ return context.getImportService().createSamples(sessionToken, uploadKey, typeCode, defaultSpaceIdentifier, spaceIdentifierOverride, experimentIdentifierOverride, updateExisting, asynchrounous, userEmail);
+ elif operation == "updateSamples":
+ return context.getImportService().updateSamples(sessionToken, uploadKey, typeCode, defaultSpaceIdentifier, spaceIdentifierOverride, experimentIdentifierOverride, asynchrounous, userEmail);
+ elif operation == "updateDataSets":
+ return context.getImportService().updateDataSets(sessionToken, uploadKey, typeCode, asynchrounous, userEmail);
+ elif operation == "createMaterials":
+ return context.getImportService().createMaterials(sessionToken, uploadKey, typeCode, updateExisting, asynchrounous, userEmail);
+ elif operation == "updateMaterials":
+ return context.getImportService().updateMaterials(sessionToken, uploadKey, typeCode, ignoreUnregistered, asynchrounous, userEmail);
+ elif operation == "generalImport":
+ return context.getImportService().generalImport(sessionToken, uploadKey, defaultSpaceIdentifier, updateExisting, asynchrounous, userEmail);
+ elif operation == "customImport":
+ return context.getImportService().customImport(sessionToken, uploadKey, customImportCode, asynchrounous, userEmail);
+
+ return None;
diff --git a/datastore_server/source/core-plugins/import-test/1/as/services/import-test/plugin.properties b/datastore_server/source/core-plugins/import-test/1/as/services/import-test/plugin.properties
new file mode 100644
index 0000000000000000000000000000000000000000..edcd58fc8199dc34bc0c90b0b5a273911b65b24f
--- /dev/null
+++ b/datastore_server/source/core-plugins/import-test/1/as/services/import-test/plugin.properties
@@ -0,0 +1,2 @@
+class = ch.ethz.sis.openbis.generic.server.asapi.v3.helper.service.JythonBasedCustomASServiceExecutor
+script-path = import-test.py
\ No newline at end of file
diff --git a/datastore_server/sourceTest/core-plugins/generic-test/1/dss/drop-boxes/test-custom-import/plugin.properties b/datastore_server/sourceTest/core-plugins/generic-test/1/dss/drop-boxes/test-custom-import/plugin.properties
new file mode 100644
index 0000000000000000000000000000000000000000..f71fcdefe5dd50af4ba8f82b6b7ebf51fedbadc1
--- /dev/null
+++ b/datastore_server/sourceTest/core-plugins/generic-test/1/dss/drop-boxes/test-custom-import/plugin.properties
@@ -0,0 +1,6 @@
+incoming-dir = ${root-dir}/incoming-test-custom-import
+incoming-dir-create = true
+incoming-data-completeness-condition = auto-detection
+top-level-data-set-handler = ch.systemsx.cisd.etlserver.registrator.api.v2.JythonTopLevelDataSetHandlerV2
+script-path = script.py
+storage-processor = ch.systemsx.cisd.etlserver.DefaultStorageProcessor
\ No newline at end of file
diff --git a/datastore_server/sourceTest/core-plugins/generic-test/1/dss/drop-boxes/test-custom-import/script.py b/datastore_server/sourceTest/core-plugins/generic-test/1/dss/drop-boxes/test-custom-import/script.py
new file mode 100644
index 0000000000000000000000000000000000000000..7fc3d060f7dbc447824223fc6530384862c0d6ec
--- /dev/null
+++ b/datastore_server/sourceTest/core-plugins/generic-test/1/dss/drop-boxes/test-custom-import/script.py
@@ -0,0 +1,5 @@
+def process(transaction):
+ data = transaction.createNewDataSet("HCS_IMAGE", transaction.getIncoming().getName())
+ data.setExperiment(transaction.getExperiment("/CISD/NEMO/EXP1"))
+ data.setPropertyValue("COMMENT", "test comment " + transaction.getIncoming().getName())
+ transaction.moveFile(transaction.getIncoming().getPath(), data)
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/AbstractFileTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/AbstractFileTest.java
index 08aa2477a357531cb2b13b031478f46bf5889111..208c4ab48e4bdf6e83a310ec5d5cd33d6199439c 100644
--- a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/AbstractFileTest.java
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/AbstractFileTest.java
@@ -27,15 +27,17 @@ public class AbstractFileTest extends SystemTestCase
{
public static final String TEST_USER = "test";
-
+
public static final String TEST_SPACE_USER = "test_space";
+ public static final String ETL_SERVER_USER = "etlserver";
+
public static final String PASSWORD = "password";
protected IGeneralInformationService gis;
protected IApplicationServerApi as;
-
+
protected IDataStoreServerApi dss;
protected String dataSetCode;
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateExperimentsImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateExperimentsImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..51fbf9a17a1385710d9aabe37f2ba1c1c1fe69ac
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateExperimentsImportTest.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier;
+
+/**
+ * @author pkupczyk
+ */
+public class CreateExperimentsImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testCreate(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+
+ String experimentCode = "TEST-IMPORT-" + UUID.randomUUID().toString();
+ ExperimentIdentifier experimentIdentifier = new ExperimentIdentifier("/TEST-SPACE/TEST-PROJECT/" + experimentCode);
+
+ ImportFile file = new ImportFile("identifier", "DESCRIPTION");
+ file.addLine(experimentIdentifier.getIdentifier(), "imported description");
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, file.toString());
+
+ Experiment experiment = getObject(sessionToken, experimentIdentifier);
+ assertNull(experiment);
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_TYPE_CODE, "SIRNA_HCS");
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "createExperiments", parameters);
+
+ experiment = getObject(sessionToken, experimentIdentifier, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported description", experiment.getProperty("DESCRIPTION"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "Experiment Batch Registration successfully performed");
+ } else
+ {
+ assertEquals("1 experiment(s) found and registered.", message);
+ assertNoEmails(timestamp);
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateMaterialsImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateMaterialsImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..145141637b71e4a0448c5f08ed7ac7d4b3b460f8
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateMaterialsImportTest.java
@@ -0,0 +1,76 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.Material;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.id.MaterialPermId;
+
+/**
+ * @author pkupczyk
+ */
+public class CreateMaterialsImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testCreate(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+
+ MaterialPermId materialPermId = new MaterialPermId("TEST-IMPORT-" + UUID.randomUUID().toString(), "VIRUS");
+
+ ImportFile file = new ImportFile("code", "DESCRIPTION");
+ file.addLine(materialPermId.getCode(), "imported description");
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, file.toString());
+
+ Material material = getObject(sessionToken, materialPermId);
+ assertNull(material);
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_TYPE_CODE, materialPermId.getTypeCode());
+ parameters.put(PARAM_UPDATE_EXISTING, false);
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "createMaterials", parameters);
+
+ material = getObject(sessionToken, materialPermId, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported description", material.getProperty("DESCRIPTION"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "Material Batch Registration successfully performed");
+ } else
+ {
+ assertEquals("Registration/update of 1 material(s) is complete.", message);
+ assertNoEmails(timestamp);
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateSamplesImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateSamplesImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..16778971b6d89f05a9da1e5fecfd9893819f5480
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateSamplesImportTest.java
@@ -0,0 +1,77 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier;
+
+/**
+ * @author pkupczyk
+ */
+public class CreateSamplesImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testCreate(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+
+ String sampleCode = "TEST-IMPORT-" + UUID.randomUUID().toString();
+ SampleIdentifier sampleIdentifier = new SampleIdentifier("/TEST-SPACE/" + sampleCode);
+
+ ImportFile file = new ImportFile("identifier", "COMMENT");
+ file.addLine(sampleIdentifier.getIdentifier(), "imported comment");
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, file.toString());
+
+ Sample sample = getObject(sessionToken, sampleIdentifier);
+ assertNull(sample);
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_TYPE_CODE, "CELL_PLATE");
+ parameters.put(PARAM_UPDATE_EXISTING, false);
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "createSamples", parameters);
+
+ sample = getObject(sessionToken, sampleIdentifier, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported comment", sample.getProperty("COMMENT"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "Sample Batch Registration successfully performed");
+ } else
+ {
+ assertEquals("Registration of 1 sample(s) is complete.", message);
+ assertNoEmails(timestamp);
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CustomImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CustomImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..1a9baee379f927ea260c67bf4189e968610a3f6c
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CustomImportTest.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.util.MultiPartContentProvider;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
+
+/**
+ * @author pkupczyk
+ */
+public class CustomImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testImport(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+
+ DataSetPermId dataSetPermId = new DataSetPermId("TEST-IMPORT-" + UUID.randomUUID().toString());
+
+ MultiPartContentProvider multiPart = new MultiPartContentProvider();
+ ContentProvider contentProvider = new StringContentProvider("test-file-content");
+ multiPart.addFilePart(TEST_UPLOAD_KEY, dataSetPermId.getPermId(), contentProvider, null);
+ multiPart.close();
+
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, multiPart);
+
+ DataSet dataSet = getObject(sessionToken, dataSetPermId);
+ assertNull(dataSet);
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_CUSTOM_IMPORT_CODE, "test-custom-import");
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "customImport", parameters);
+
+ dataSet = getObject(sessionToken, dataSetPermId, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("test comment " + dataSetPermId.getPermId(), dataSet.getProperty("COMMENT"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "Custom import successfully performed");
+ } else
+ {
+ assertEquals("Import successfully completed.", message);
+ assertNoEmails(timestamp);
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/GeneralImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/GeneralImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..275e704d0607e06fb7e8710def24af45ec526b48
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/GeneralImportTest.java
@@ -0,0 +1,111 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.commons.io.FileUtils;
+import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.util.BytesContentProvider;
+import org.eclipse.jetty.client.util.MultiPartContentProvider;
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.Material;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.delete.MaterialDeletionOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.id.IMaterialId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.id.MaterialPermId;
+import ch.systemsx.cisd.common.utilities.TestResources;
+
+/**
+ * @author pkupczyk
+ */
+public class GeneralImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testImport(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+
+ MaterialPermId materialPermId1 = new MaterialPermId("TEST-IMPORT-1", "VIRUS");
+ MaterialPermId materialPermId2 = new MaterialPermId("TEST-IMPORT-2", "VIRUS");
+
+ try
+ {
+ deleteMaterials(sessionToken, materialPermId1, materialPermId2);
+
+ Material material1 = getObject(sessionToken, materialPermId1);
+ assertNull(material1);
+
+ Material material2 = getObject(sessionToken, materialPermId2);
+ assertNull(material2);
+
+ TestResources resources = new TestResources(getClass());
+ File materialsFile = resources.getResourceFile("materials_excel_97_2003.xls");
+
+ MultiPartContentProvider multiPart = new MultiPartContentProvider();
+ ContentProvider contentProvider = new BytesContentProvider(FileUtils.readFileToByteArray(materialsFile));
+ multiPart.addFilePart(TEST_UPLOAD_KEY, materialsFile.getName(), contentProvider, null);
+ multiPart.close();
+
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, multiPart);
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_UPDATE_EXISTING, false);
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "generalImport", parameters);
+
+ material1 = getObject(sessionToken, materialPermId1, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported description 1", material1.getProperty("DESCRIPTION"));
+
+ material2 = getObject(sessionToken, materialPermId2, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported description 2", material2.getProperty("DESCRIPTION"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "General Batch Import successfully performed");
+ } else
+ {
+ assertEquals("Registration/update of 2 material(s) is complete.\nRegistration of 0 sample(s) is complete.", message);
+ assertNoEmails(timestamp);
+ }
+ } finally
+ {
+ deleteMaterials(sessionToken, materialPermId1, materialPermId2);
+ }
+ }
+
+ private void deleteMaterials(String sessionToken, IMaterialId... materialIds)
+ {
+ MaterialDeletionOptions options = new MaterialDeletionOptions();
+ options.setReason("cleanup");
+ as.deleteMaterials(sessionToken, Arrays.asList(materialIds), options);
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/GeneralImportTestResources/materials_excel_97_2003.xls b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/GeneralImportTestResources/materials_excel_97_2003.xls
new file mode 100644
index 0000000000000000000000000000000000000000..3e18171654156d191bf5dd8f036f415cb1a003d4
Binary files /dev/null and b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/GeneralImportTestResources/materials_excel_97_2003.xls differ
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/ObjectsImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/ObjectsImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..b41d7e9b67760a962e23299bd4b5fa35e374b9dd
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/ObjectsImportTest.java
@@ -0,0 +1,261 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.TimeoutException;
+
+import org.eclipse.jetty.client.HttpClient;
+import org.eclipse.jetty.client.api.ContentProvider;
+import org.eclipse.jetty.client.api.ContentResponse;
+import org.eclipse.jetty.client.api.Request;
+import org.eclipse.jetty.client.util.MultiPartContentProvider;
+import org.eclipse.jetty.client.util.StringContentProvider;
+import org.eclipse.jetty.http.HttpMethod;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.DataProvider;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.id.IObjectId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IModificationDateHolder;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.IDataSetId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.IExperimentId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.fetchoptions.MaterialFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.id.IMaterialId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.ISampleId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.service.CustomASServiceExecutionOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.service.id.CustomASServiceCode;
+import ch.ethz.sis.openbis.systemtest.asapi.v3.util.EmailUtil;
+import ch.ethz.sis.openbis.systemtest.asapi.v3.util.EmailUtil.Email;
+import ch.systemsx.cisd.common.http.JettyHttpClientFactory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.util.TestInstanceHostUtils;
+
+/**
+ * @author pkupczyk
+ */
+public class ObjectsImportTest extends AbstractFileTest
+{
+
+ protected static final String SERVICE_URL = TestInstanceHostUtils.getOpenBISUrl() + "/openbis/upload";
+
+ protected static final String TEST_UPLOAD_KEY = "test-import";
+
+ protected static final String TEST_EMAIL = "test@email.com";
+
+ protected static final long DEFAULT_TIMEOUT = 30000;
+
+ protected static final String FALSE_TRUE_PROVIDER = "sync-async";
+
+ protected static final String PARAM_UPLOAD_KEY = "uploadKey";
+
+ protected static final String PARAM_TYPE_CODE = "typeCode";
+
+ protected static final String PARAM_ASYNC = "async";
+
+ protected static final String PARAM_USER_EMAIL = "userEmail";
+
+ protected static final String PARAM_DEFAULT_SPACE_IDENTIFIER = "defaultSpaceIdentifier";
+
+ protected static final String PARAM_SPACE_IDENTIFIER_OVERRIDE = "spaceIdentifierOverride";
+
+ protected static final String PARAM_EXPERIMENT_IDENTIFIER_OVERRIDE = "experimentIdentifierOverride";
+
+ protected static final String PARAM_UPDATE_EXISTING = "updateExisting";
+
+ protected static final String PARAM_IGNORE_UNREGISTERED = "ignoreUnregistered";
+
+ protected static final String PARAM_CUSTOM_IMPORT_CODE = "customImportCode";
+
+ protected IApplicationServerApi as;
+
+ @BeforeClass
+ protected void beforeClass() throws Exception
+ {
+ super.beforeClass();
+ as = ServiceProvider.getV3ApplicationService();
+ }
+
+ protected ContentResponse uploadFiles(String sessionToken, String uploadSessionKey, MultiPartContentProvider multiPart)
+ throws InterruptedException, TimeoutException, ExecutionException
+ {
+ HttpClient client = JettyHttpClientFactory.getHttpClient();
+ Request request = client.newRequest(SERVICE_URL).method(HttpMethod.POST);
+ request.param("sessionID", sessionToken);
+ request.param("sessionKeysNumber", "1");
+ request.param("sessionKey_0", uploadSessionKey);
+ request.content(multiPart);
+
+ return request.send();
+ }
+
+ protected ContentResponse uploadFiles(String sessionToken, String uploadSessionKey, String... filesContent)
+ throws InterruptedException, TimeoutException, ExecutionException
+ {
+ MultiPartContentProvider multiPart = new MultiPartContentProvider();
+
+ for (int i = 0; i < filesContent.length; i++)
+ {
+ ContentProvider contentProvider = new StringContentProvider(filesContent[i]);
+
+ String fieldName = uploadSessionKey + "_" + i;
+ String fileName = "fileName_" + i;
+ multiPart.addFilePart(fieldName, fileName, contentProvider, null);
+ }
+
+ multiPart.close();
+
+ return uploadFiles(sessionToken, uploadSessionKey, multiPart);
+ }
+
+ protected String executeImport(String sessionToken, String operation, Map<String, Object> parameters)
+ {
+ CustomASServiceCode serviceId = new CustomASServiceCode("import-test");
+ CustomASServiceExecutionOptions options = new CustomASServiceExecutionOptions();
+ options.withParameter("operation", operation);
+ for (String name : parameters.keySet())
+ {
+ options.withParameter(name, parameters.get(name));
+ }
+ return (String) as.executeCustomASService(sessionToken, serviceId, options);
+ }
+
+ protected <T extends IModificationDateHolder> T getObject(String sessionToken, IObjectId objectId)
+ {
+ return getObject(sessionToken, objectId, 0, 0);
+ }
+
+ protected <T extends IModificationDateHolder> T getObject(String sessionToken, IObjectId objectId, long modifiedAfterTimestamp,
+ long timeoutAfterMillis)
+ {
+ long startMillis = System.currentTimeMillis();
+
+ while (true)
+ {
+ Map<IObjectId, T> objects = getObjects(sessionToken, objectId);
+ T object = objects.get(objectId);
+
+ if (object != null && object.getModificationDate() != null && object.getModificationDate().getTime() >= modifiedAfterTimestamp)
+ {
+ return object;
+ }
+
+ if (timeoutAfterMillis > 0 && System.currentTimeMillis() < startMillis + timeoutAfterMillis)
+ {
+ try
+ {
+ Thread.sleep(100);
+ } catch (InterruptedException e)
+ {
+ throw new RuntimeException(e);
+ }
+ } else
+ {
+ return null;
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private <K, V> Map<K, V> getObjects(String sessionToken, IObjectId objectId)
+ {
+ if (objectId instanceof IExperimentId)
+ {
+ ExperimentFetchOptions fo = new ExperimentFetchOptions();
+ fo.withProperties();
+ return (Map<K, V>) as.getExperiments(sessionToken, Arrays.asList((IExperimentId) objectId), (ExperimentFetchOptions) fo);
+ } else if (objectId instanceof ISampleId)
+ {
+ SampleFetchOptions fo = new SampleFetchOptions();
+ fo.withProperties();
+ return (Map<K, V>) as.getSamples(sessionToken, Arrays.asList((ISampleId) objectId), (SampleFetchOptions) fo);
+ } else if (objectId instanceof IDataSetId)
+ {
+ DataSetFetchOptions fo = new DataSetFetchOptions();
+ fo.withProperties();
+ return (Map<K, V>) as.getDataSets(sessionToken, Arrays.asList((IDataSetId) objectId), (DataSetFetchOptions) fo);
+ } else if (objectId instanceof IMaterialId)
+ {
+ MaterialFetchOptions fo = new MaterialFetchOptions();
+ fo.withProperties();
+ return (Map<K, V>) as.getMaterials(sessionToken, Arrays.asList((IMaterialId) objectId), (MaterialFetchOptions) fo);
+ } else
+ {
+ throw new IllegalArgumentException("Unsupported object id " + objectId);
+ }
+ }
+
+ protected void assertNoEmails(long timestamp)
+ {
+ Email latestEmail = EmailUtil.findLatestEmail();
+ assertTrue("Timestamp: " + timestamp + ", Latest email: " + latestEmail, latestEmail == null || latestEmail.timestamp < timestamp);
+ }
+
+ protected void assertEmail(long timestamp, String expectedEmail, String expectedSubject)
+ {
+ Email latestEmail = EmailUtil.findLatestEmail();
+ assertTrue("Timestamp: " + timestamp + ", Latest email: " + latestEmail, latestEmail != null && latestEmail.timestamp >= timestamp);
+ assertEquals(expectedEmail, latestEmail.to);
+ assertTrue(latestEmail.subject, latestEmail.subject.contains(expectedSubject));
+ }
+
+ public static class ImportFile
+ {
+
+ private List<String> columns;
+
+ private List<List<String>> lines = new ArrayList<List<String>>();
+
+ public ImportFile(String... columns)
+ {
+ this.columns = Arrays.asList(columns);
+ }
+
+ public void addLine(String... values)
+ {
+ lines.add(Arrays.asList(values));
+ }
+
+ @Override
+ public String toString()
+ {
+ StringBuilder content = new StringBuilder();
+ content.append(String.join("\t", columns) + "\n");
+
+ for (List<String> line : lines)
+ {
+ content.append(String.join("\t", line) + "\n");
+ }
+
+ return content.toString();
+ }
+ }
+
+ @DataProvider(name = FALSE_TRUE_PROVIDER)
+ public static Object[][] provideFalseTrue()
+ {
+ return new Object[][] { { false }, { true } };
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateDataSetsImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateDataSetsImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..017f932202864e280d71a5d096d319ae1f137a70
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateDataSetsImportTest.java
@@ -0,0 +1,106 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSetKind;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.create.DataSetCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.create.PhysicalDataCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.FileFormatTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.ProprietaryStorageFormatPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.RelativeLocationLocatorTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.datastore.id.DataStorePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier;
+
+/**
+ * @author pkupczyk
+ */
+public class UpdateDataSetsImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testUpdate(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+ String etlServerSessionToken = as.login(ETL_SERVER_USER, PASSWORD);
+
+ DataSetPermId dataSetPermId = new DataSetPermId("TEST-IMPORT-" + UUID.randomUUID().toString());
+
+ PhysicalDataCreation physicalCreation = new PhysicalDataCreation();
+ physicalCreation.setLocation("test/location/" + dataSetPermId.getPermId());
+ physicalCreation.setFileFormatTypeId(new FileFormatTypePermId("TIFF"));
+ physicalCreation.setLocatorTypeId(new RelativeLocationLocatorTypePermId());
+ physicalCreation.setStorageFormatId(new ProprietaryStorageFormatPermId());
+
+ DataSetCreation creation = new DataSetCreation();
+ creation.setCode(dataSetPermId.getPermId());
+ creation.setDataSetKind(DataSetKind.PHYSICAL);
+ creation.setTypeId(new EntityTypePermId("HCS_IMAGE"));
+ creation.setExperimentId(new ExperimentIdentifier("/TEST-SPACE/TEST-PROJECT/EXP-SPACE-TEST"));
+ creation.setDataStoreId(new DataStorePermId("STANDARD"));
+ creation.setPhysicalData(physicalCreation);
+ creation.setProperty("COMMENT", "initial comment");
+
+ DataSet dataSet = getObject(sessionToken, dataSetPermId);
+ assertNull(dataSet);
+
+ as.createDataSets(etlServerSessionToken, Arrays.asList(creation));
+
+ dataSet = getObject(sessionToken, dataSetPermId);
+ assertEquals("initial comment", dataSet.getProperty("COMMENT"));
+
+ ImportFile file = new ImportFile("code", "COMMENT");
+ file.addLine(dataSetPermId.getPermId(), "imported comment");
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, file.toString());
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_TYPE_CODE, "HCS_IMAGE");
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "updateDataSets", parameters);
+
+ dataSet = getObject(sessionToken, dataSetPermId, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported comment", dataSet.getProperty("COMMENT"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "Data Set Batch Update successfully performed");
+ } else
+ {
+ assertEquals("1 data set(s) found and registered.", message);
+ assertNoEmails(timestamp);
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateExperimentsImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateExperimentsImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..b2b28709521454188430fcab3cdf09e2c7450ead
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateExperimentsImportTest.java
@@ -0,0 +1,91 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.create.ExperimentCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.id.ProjectIdentifier;
+
+/**
+ * @author pkupczyk
+ */
+public class UpdateExperimentsImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testUpdate(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+
+ String experimentCode = "TEST-IMPORT-" + UUID.randomUUID().toString();
+ ExperimentIdentifier experimentIdentifier = new ExperimentIdentifier("/TEST-SPACE/TEST-PROJECT/" + experimentCode);
+
+ ExperimentCreation creation = new ExperimentCreation();
+ creation.setCode(experimentCode);
+ creation.setTypeId(new EntityTypePermId("SIRNA_HCS"));
+ creation.setProjectId(new ProjectIdentifier("/TEST-SPACE/TEST-PROJECT"));
+ creation.setProperty("DESCRIPTION", "initial value");
+
+ Experiment experiment = getObject(sessionToken, experimentIdentifier);
+ assertNull(experiment);
+
+ as.createExperiments(sessionToken, Arrays.asList(creation));
+
+ experiment = getObject(sessionToken, experimentIdentifier);
+ assertEquals("initial value", experiment.getProperty("DESCRIPTION"));
+
+ ImportFile file = new ImportFile("identifier", "DESCRIPTION");
+ file.addLine(experimentIdentifier.getIdentifier(), "imported description");
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, file.toString());
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_TYPE_CODE, "SIRNA_HCS");
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "updateExperiments", parameters);
+
+ experiment = getObject(sessionToken, experimentIdentifier, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported description", experiment.getProperty("DESCRIPTION"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "Experiment Batch Update successfully performed");
+ } else
+ {
+ assertEquals("Update of 1 experiment(s) is complete.", message);
+ assertNoEmails(timestamp);
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateMaterialsImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateMaterialsImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..a049a46f0850dbe90039a6edc9294ff11834f26f
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateMaterialsImportTest.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.Material;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.create.MaterialCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.material.id.MaterialPermId;
+
+/**
+ * @author pkupczyk
+ */
+public class UpdateMaterialsImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testUpdate(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+
+ MaterialPermId materialPermId = new MaterialPermId("TEST-IMPORT-" + UUID.randomUUID().toString(), "VIRUS");
+
+ MaterialCreation creation = new MaterialCreation();
+ creation.setCode(materialPermId.getCode());
+ creation.setTypeId(new EntityTypePermId(materialPermId.getTypeCode()));
+ creation.setProperty("DESCRIPTION", "initial description");
+
+ Material material = getObject(sessionToken, materialPermId);
+ assertNull(material);
+
+ as.createMaterials(sessionToken, Arrays.asList(creation));
+
+ material = getObject(sessionToken, materialPermId);
+ assertEquals("initial description", material.getProperty("DESCRIPTION"));
+
+ ImportFile file = new ImportFile("code", "DESCRIPTION");
+ file.addLine(materialPermId.getCode(), "imported description");
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, file.toString());
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_TYPE_CODE, materialPermId.getTypeCode());
+ parameters.put(PARAM_IGNORE_UNREGISTERED, false);
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "updateMaterials", parameters);
+
+ material = getObject(sessionToken, materialPermId, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported description", material.getProperty("DESCRIPTION"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "Material Batch Update successfully performed");
+ } else
+ {
+ assertEquals("1 material(s) updated.", message);
+ assertNoEmails(timestamp);
+ }
+ }
+
+}
diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateSamplesImportTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateSamplesImportTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..c6b1e6cd364bb4269628ccd2a4c0e2302211bb21
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/UpdateSamplesImportTest.java
@@ -0,0 +1,92 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3;
+
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.UUID;
+
+import org.testng.annotations.Test;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.create.SampleCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.id.SpacePermId;
+
+/**
+ * @author pkupczyk
+ */
+public class UpdateSamplesImportTest extends ObjectsImportTest
+{
+
+ @Test(dataProvider = FALSE_TRUE_PROVIDER)
+ public void testUpdate(boolean async) throws Exception
+ {
+ String sessionToken = as.login(TEST_USER, PASSWORD);
+
+ String sampleCode = "TEST-IMPORT-" + UUID.randomUUID().toString();
+ SampleIdentifier sampleIdentifier = new SampleIdentifier("/TEST-SPACE/" + sampleCode);
+
+ SampleCreation creation = new SampleCreation();
+ creation.setCode(sampleCode);
+ creation.setSpaceId(new SpacePermId("TEST-SPACE"));
+ creation.setTypeId(new EntityTypePermId("CELL_PLATE"));
+ creation.setProperty("COMMENT", "initial comment");
+
+ Sample sample = getObject(sessionToken, sampleIdentifier);
+ assertNull(sample);
+
+ as.createSamples(sessionToken, Arrays.asList(creation));
+
+ sample = getObject(sessionToken, sampleIdentifier);
+ assertEquals("initial comment", sample.getProperty("COMMENT"));
+
+ ImportFile file = new ImportFile("identifier", "COMMENT");
+ file.addLine(sampleIdentifier.getIdentifier(), "imported comment");
+ uploadFiles(sessionToken, TEST_UPLOAD_KEY, file.toString());
+
+ Map<String, Object> parameters = new HashMap<String, Object>();
+ parameters.put(PARAM_UPLOAD_KEY, TEST_UPLOAD_KEY);
+ parameters.put(PARAM_TYPE_CODE, "CELL_PLATE");
+ parameters.put(PARAM_UPDATE_EXISTING, false);
+ parameters.put(PARAM_ASYNC, async);
+
+ if (async)
+ {
+ parameters.put(PARAM_USER_EMAIL, TEST_EMAIL);
+ }
+
+ long timestamp = System.currentTimeMillis();
+ String message = executeImport(sessionToken, "updateSamples", parameters);
+
+ sample = getObject(sessionToken, sampleIdentifier, timestamp, DEFAULT_TIMEOUT);
+ assertEquals("imported comment", sample.getProperty("COMMENT"));
+
+ if (async)
+ {
+ assertEquals("When the import is complete the confirmation or failure report will be sent by email.", message);
+ assertEmail(timestamp, TEST_EMAIL, "Sample Batch Update successfully performed");
+ } else
+ {
+ assertEquals("Update of 1 sample(s) is complete.", message);
+ assertNoEmails(timestamp);
+ }
+ }
+
+}
diff --git a/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/CustomASServiceScriptRunnerFactory.java b/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/CustomASServiceScriptRunnerFactory.java
index 7837c8e87a79da7faa2a7028d3326d21e73df1f8..0954144dde8b831564941cfc1cfb904b9238fcc0 100644
--- a/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/CustomASServiceScriptRunnerFactory.java
+++ b/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/CustomASServiceScriptRunnerFactory.java
@@ -20,6 +20,7 @@ import java.io.Serializable;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.service.CustomASServiceExecutionOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.plugin.service.IImportService;
import ch.ethz.sis.openbis.generic.asapi.v3.plugin.service.context.ServiceContext;
import ch.systemsx.cisd.common.jython.JythonUtils;
import ch.systemsx.cisd.common.jython.evaluator.Evaluator;
@@ -35,10 +36,14 @@ class CustomASServiceScriptRunnerFactory implements IScriptRunnerFactory
private final IApplicationServerApi applicationService;
- public CustomASServiceScriptRunnerFactory(String scriptPath, IApplicationServerApi applicationService)
+ private final IImportService importService;
+
+ public CustomASServiceScriptRunnerFactory(String scriptPath, IApplicationServerApi applicationService,
+ IImportService importService)
{
this.scriptPath = scriptPath;
this.applicationService = applicationService;
+ this.importService = importService;
Evaluator.getFactory().initialize();
}
@@ -58,7 +63,7 @@ class CustomASServiceScriptRunnerFactory implements IScriptRunnerFactory
{
IJythonEvaluator evaluator = Evaluator.getFactory().create("", pythonPath, scriptPath, null, scriptString, false);
String sessionToken = context.getSessionToken();
- ExecutionContext executionContext = new ExecutionContext(sessionToken, applicationService);
+ ExecutionContext executionContext = new ExecutionContext(sessionToken, applicationService, importService);
return new ServiceScriptRunner(evaluator, executionContext);
} catch (EvaluatorException ex)
{
@@ -105,10 +110,13 @@ class CustomASServiceScriptRunnerFactory implements IScriptRunnerFactory
private final IApplicationServerApi applicationService;
- ExecutionContext(String sessionToken, IApplicationServerApi applicationService)
+ private final IImportService importService;
+
+ ExecutionContext(String sessionToken, IApplicationServerApi applicationService, IImportService importService)
{
this.sessionToken = sessionToken;
this.applicationService = applicationService;
+ this.importService = importService;
}
public String getSessionToken()
@@ -120,5 +128,10 @@ class CustomASServiceScriptRunnerFactory implements IScriptRunnerFactory
{
return applicationService;
}
+
+ public IImportService getImportService()
+ {
+ return importService;
+ }
}
}
diff --git a/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/ImportService.java b/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/ImportService.java
new file mode 100644
index 0000000000000000000000000000000000000000..1866d55cff75ec3f964e047c617af93b68c31dcf
--- /dev/null
+++ b/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/ImportService.java
@@ -0,0 +1,212 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.server.asapi.v3.helper.service;
+
+import java.util.Iterator;
+import java.util.List;
+
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Component;
+
+import ch.ethz.sis.openbis.generic.asapi.v3.plugin.service.IImportService;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.openbis.generic.server.ICustomImportService;
+import ch.systemsx.cisd.openbis.generic.server.IEntityImportService;
+import ch.systemsx.cisd.openbis.generic.shared.ResourceNames;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.BatchRegistrationResult;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExperimentType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
+
+/**
+ * @author pkupczyk
+ */
+@Component(value = ResourceNames.IMPORT_SERVICE)
+public class ImportService implements IImportService
+{
+
+ @Autowired
+ private IEntityImportService entityImportService;
+
+ @Autowired
+ private ICustomImportService customImportService;
+
+ @Override
+ public String createExperiments(String sessionToken, String uploadKey, String experimentTypeCode, boolean async, String userEmail)
+ {
+ check(sessionToken, uploadKey, experimentTypeCode, async, userEmail);
+
+ ExperimentType experimentType = new ExperimentType();
+ experimentType.setCode(experimentTypeCode);
+
+ List<BatchRegistrationResult> results = entityImportService.registerExperiments(experimentType, uploadKey, async, userEmail);
+ return translateResults(results);
+ }
+
+ @Override
+ public String updateExperiments(String sessionToken, String uploadKey, String experimentTypeCode, boolean async, String userEmail)
+ {
+ check(sessionToken, uploadKey, experimentTypeCode, async, userEmail);
+
+ ExperimentType experimentType = new ExperimentType();
+ experimentType.setCode(experimentTypeCode);
+
+ List<BatchRegistrationResult> results = entityImportService.updateExperiments(experimentType, uploadKey, async, userEmail);
+ return translateResults(results);
+ }
+
+ @Override
+ public String createSamples(String sessionToken, String uploadKey, String sampleTypeCode, String defaultSpaceIdentifier,
+ String spaceIdentifierOverride, String experimentIdentifierOverride, boolean updateExisting, boolean async, String userEmail)
+ {
+ check(sessionToken, uploadKey, sampleTypeCode, async, userEmail);
+
+ SampleType sampleType = new SampleType();
+ sampleType.setCode(sampleTypeCode);
+
+ List<BatchRegistrationResult> results = entityImportService.registerSamplesWithSilentOverrides(sampleType, spaceIdentifierOverride,
+ experimentIdentifierOverride, uploadKey, async, userEmail, defaultSpaceIdentifier, updateExisting);
+ return translateResults(results);
+ }
+
+ @Override
+ public String updateSamples(String sessionToken, String uploadKey, String sampleTypeCode, String defaultSpaceIdentifier,
+ String spaceIdentifierOverride, String experimentIdentifierOverride, boolean async, String userEmail)
+ {
+ check(sessionToken, uploadKey, sampleTypeCode, async, userEmail);
+
+ SampleType sampleType = new SampleType();
+ sampleType.setCode(sampleTypeCode);
+
+ List<BatchRegistrationResult> results = entityImportService.updateSamplesWithSilentOverrides(sampleType, spaceIdentifierOverride,
+ experimentIdentifierOverride, uploadKey, async, userEmail, defaultSpaceIdentifier);
+ return translateResults(results);
+ }
+
+ @Override
+ public String updateDataSets(String sessionToken, String uploadKey, String dataSetTypeCode, boolean async, String userEmail)
+ {
+ check(sessionToken, uploadKey, dataSetTypeCode, async, userEmail);
+
+ DataSetType dataSetType = new DataSetType();
+ dataSetType.setCode(dataSetTypeCode);
+
+ List<BatchRegistrationResult> results = entityImportService.updateDataSets(dataSetType, uploadKey, async, userEmail);
+ return translateResults(results);
+ }
+
+ @Override
+ public String createMaterials(String sessionToken, String uploadKey, String materialTypeCode, boolean updateExisting, boolean async,
+ String userEmail)
+ {
+ check(sessionToken, uploadKey, materialTypeCode, async, userEmail);
+
+ MaterialType materialType = new MaterialType();
+ materialType.setCode(materialTypeCode);
+
+ List<BatchRegistrationResult> results = entityImportService.registerMaterials(materialType, updateExisting, uploadKey, async, userEmail);
+ return translateResults(results);
+ }
+
+ @Override
+ public String updateMaterials(String sessionToken, String uploadKey, String materialTypeCode, boolean ignoreUnregistered, boolean async,
+ String userEmail)
+ {
+ check(sessionToken, uploadKey, materialTypeCode, async, userEmail);
+
+ MaterialType materialType = new MaterialType();
+ materialType.setCode(materialTypeCode);
+
+ List<BatchRegistrationResult> results = entityImportService.updateMaterials(materialType, uploadKey, ignoreUnregistered, async, userEmail);
+ return translateResults(results);
+ }
+
+ @Override
+ public String generalImport(String sessionToken, String uploadKey, String defaultSpaceIdentifier, boolean updateExisting, boolean async,
+ String userEmail)
+ {
+ check(sessionToken, uploadKey, async, userEmail);
+
+ List<BatchRegistrationResult> results =
+ entityImportService.registerOrUpdateSamplesAndMaterials(uploadKey, defaultSpaceIdentifier, updateExisting, async, userEmail);
+ return translateResults(results);
+ }
+
+ @Override
+ public String customImport(String sessionToken, String uploadKey, String customImportCode, boolean async, String userEmail)
+ {
+ check(sessionToken, uploadKey, async, userEmail);
+
+ if (customImportCode == null)
+ {
+ throw new UserFailureException("Custom import code cannot be null");
+ }
+
+ List<BatchRegistrationResult> results =
+ customImportService.performCustomImport(uploadKey, customImportCode, async, userEmail);
+ return translateResults(results);
+ }
+
+ protected void check(String sessionToken, String uploadKey, String typeCode, boolean async, String userEmail)
+ {
+ check(sessionToken, uploadKey, async, userEmail);
+
+ if (typeCode == null)
+ {
+ throw new UserFailureException("Type code cannot be null");
+ }
+ }
+
+ protected void check(String sessionToken, String uploadKey, boolean async, String userEmail)
+ {
+ if (sessionToken == null)
+ {
+ throw new UserFailureException("Session token cannot be null");
+ }
+ if (uploadKey == null)
+ {
+ throw new UserFailureException("Upload key cannot be null");
+ }
+ if (async && userEmail == null)
+ {
+ throw new UserFailureException("User email cannot be null for an asynchronous import");
+ }
+ }
+
+ protected String translateResults(List<BatchRegistrationResult> results)
+ {
+ StringBuilder message = new StringBuilder();
+ Iterator<BatchRegistrationResult> iter = results.iterator();
+
+ while (iter.hasNext())
+ {
+ BatchRegistrationResult result = iter.next();
+ if (result != null)
+ {
+ message.append(result.getMessage());
+ if (iter.hasNext())
+ {
+ message.append("\n");
+ }
+ }
+ }
+
+ return message.toString();
+ }
+
+}
diff --git a/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/JythonBasedCustomASServiceExecutor.java b/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/JythonBasedCustomASServiceExecutor.java
index 08cbd921a0568ed1eb12b28883650f8d90e1b91d..3c56376af5a853328785a2115bd823b005d20b14 100644
--- a/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/JythonBasedCustomASServiceExecutor.java
+++ b/openbis/source/java/ch/ethz/sis/openbis/generic/server/asapi/v3/helper/service/JythonBasedCustomASServiceExecutor.java
@@ -51,7 +51,8 @@ public class JythonBasedCustomASServiceExecutor implements ICustomASServiceExecu
{
if (factory == null)
{
- factory = new CustomASServiceScriptRunnerFactory(scriptPath, CommonServiceProvider.getApplicationServerApi());
+ factory = new CustomASServiceScriptRunnerFactory(scriptPath, CommonServiceProvider.getApplicationServerApi(),
+ CommonServiceProvider.getImportService());
}
return factory;
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/client/web/server/CommonClientService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/client/web/server/CommonClientService.java
index 2616f00df8bfc246220b71e9081815eb76ed75ec..6eb4083dcc99d0c9ea60362685beb0aeca5e9895 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/client/web/server/CommonClientService.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/client/web/server/CommonClientService.java
@@ -123,6 +123,7 @@ import ch.systemsx.cisd.openbis.generic.client.web.server.translator.SearchableE
import ch.systemsx.cisd.openbis.generic.client.web.server.translator.SearchableSearchDomainTranslator;
import ch.systemsx.cisd.openbis.generic.client.web.server.translator.UserFailureExceptionTranslator;
import ch.systemsx.cisd.openbis.generic.client.web.server.util.TSVRenderer;
+import ch.systemsx.cisd.openbis.generic.server.ICustomImportService;
import ch.systemsx.cisd.openbis.generic.shared.ICommonServer;
import ch.systemsx.cisd.openbis.generic.shared.IServer;
import ch.systemsx.cisd.openbis.generic.shared.api.v1.dto.SearchDomain;
@@ -241,7 +242,7 @@ import ch.systemsx.cisd.openbis.generic.shared.util.WebClientConfigUtils;
* @author Franz-Josef Elmer
*/
public final class CommonClientService extends AbstractClientService implements
- ICommonClientService
+ ICommonClientService, ICustomImportService
{
@Resource(name = "registration-queue")
private ConsumerQueue asyncRegistrationQueue;
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServiceProvider.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServiceProvider.java
index b311ae93581827d73962be8973c7ee904c168b4a..074dae2a3ad1aae293a63da77950827cbb76bb91 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServiceProvider.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/CommonServiceProvider.java
@@ -19,6 +19,7 @@ package ch.systemsx.cisd.openbis.generic.server;
import org.springframework.context.ApplicationContext;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.plugin.service.IImportService;
import ch.ethz.sis.openbis.generic.server.asapi.v3.ApplicationServerApi;
import ch.systemsx.cisd.common.mail.IMailClient;
import ch.systemsx.cisd.common.mail.MailClient;
@@ -47,6 +48,11 @@ public class CommonServiceProvider
.getBean(ResourceNames.COMMON_SERVER);
}
+ public static IImportService getImportService()
+ {
+ return (IImportService) applicationContext.getBean(ResourceNames.IMPORT_SERVICE);
+ }
+
public static IDAOFactory getDAOFactory()
{
return (IDAOFactory) applicationContext.getBean("dao-factory");
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ICustomImportService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ICustomImportService.java
new file mode 100644
index 0000000000000000000000000000000000000000..1a2a9e0cb6b6d29f2491f9e765ee7252202d4a9c
--- /dev/null
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ICustomImportService.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.generic.server;
+
+import java.util.List;
+
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.BatchRegistrationResult;
+
+/**
+ * @author pkupczyk
+ */
+public interface ICustomImportService
+{
+
+ public List<BatchRegistrationResult> performCustomImport(String sessionKey, String customImportCode, boolean async, String userEmail);
+
+}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/IEntityImportService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/IEntityImportService.java
new file mode 100644
index 0000000000000000000000000000000000000000..f3015cc1eee57264d9a7305b449757bd7991a16d
--- /dev/null
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/IEntityImportService.java
@@ -0,0 +1,55 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.generic.server;
+
+import java.util.List;
+
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.BatchRegistrationResult;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExperimentType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
+
+/**
+ * @author pkupczyk
+ */
+public interface IEntityImportService
+{
+
+ public List<BatchRegistrationResult> registerExperiments(ExperimentType experimentType, String sessionKey, boolean async, String userEmail);
+
+ public List<BatchRegistrationResult> updateExperiments(ExperimentType experimentType, String sessionKey, boolean async, String userEmail);
+
+ public List<BatchRegistrationResult> registerSamplesWithSilentOverrides(SampleType sampleType, String spaceIdentifierSilentOverrideOrNull,
+ String experimentIdentifierSilentOverrideOrNull, String sessionKey, boolean async, String userEmail, String defaultGroupIdentifier,
+ boolean updateExisting);
+
+ public List<BatchRegistrationResult> updateSamplesWithSilentOverrides(SampleType sampleType, String spaceIdentifierSilentOverrideOrNull,
+ String experimentIdentifierSilentOverrideOrNull, String sessionKey, boolean async, String userEmail, String defaultGroupIdentifier);
+
+ public List<BatchRegistrationResult> updateDataSets(DataSetType dataSetType, String sessionKey, boolean async, String userEmail);
+
+ public List<BatchRegistrationResult> registerMaterials(MaterialType materialType, boolean updateExisting, final String sessionKey, boolean async,
+ String userEmail);
+
+ public List<BatchRegistrationResult> updateMaterials(MaterialType materialType, String sessionKey, boolean ignoreUnregistered,
+ boolean async, String userEmail);
+
+ public List<BatchRegistrationResult> registerOrUpdateSamplesAndMaterials(String sessionKey, String defaultGroupIdentifier, boolean updateExisting,
+ boolean async, String userEmail);
+
+}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SampleTable.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SampleTable.java
index 2432d8f1e8f2d48d248e0c6d9131de54d8072f82..abb8ceb92a739faf722973bcd6621aaf609e19a6 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SampleTable.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SampleTable.java
@@ -316,6 +316,8 @@ public final class SampleTable extends AbstractSampleBusinessObject implements I
{
relationshipService.assignSampleToSpace(session, sample, sample.getSpace());
}
+
+ RelationshipUtils.updateModificationDateAndModifier(sample, session, getTransactionTimeStamp());
}
/**
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/ResourceNames.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/ResourceNames.java
index 03492b312079b84558098fe65c3523c72c1577a5..afb075c8cdeca7e570fc0280390f871633ec9530 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/ResourceNames.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/ResourceNames.java
@@ -60,6 +60,8 @@ public final class ResourceNames
public final static String COMMON_SERVICE = "common-service";
public final static String COMMON_SERVER = "common-server";
+
+ public final static String IMPORT_SERVICE = "import-service";
public final static String TRACKING_SERVER = "tracking-server";
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/plugin/generic/client/web/server/GenericClientService.java b/openbis/source/java/ch/systemsx/cisd/openbis/plugin/generic/client/web/server/GenericClientService.java
index d04c1aa2a9abb73875dd14e4d0615d9c7775629b..c86812246e5a43f46a8cf782f1bd18e7144ac8bb 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/plugin/generic/client/web/server/GenericClientService.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/plugin/generic/client/web/server/GenericClientService.java
@@ -49,6 +49,7 @@ import ch.systemsx.cisd.openbis.generic.client.web.server.UploadedFilesBean;
import ch.systemsx.cisd.openbis.generic.client.web.server.queue.ConsumerQueue;
import ch.systemsx.cisd.openbis.generic.client.web.server.queue.ConsumerTask;
import ch.systemsx.cisd.openbis.generic.client.web.server.translator.UserFailureExceptionTranslator;
+import ch.systemsx.cisd.openbis.generic.server.IEntityImportService;
import ch.systemsx.cisd.openbis.generic.server.dataaccess.db.exception.SampleUniqueCodeViolationException;
import ch.systemsx.cisd.openbis.generic.server.dataaccess.db.exception.SampleUniqueCodeViolationExceptionAbstract;
import ch.systemsx.cisd.openbis.generic.shared.Constants;
@@ -110,7 +111,7 @@ import ch.systemsx.cisd.openbis.plugin.generic.shared.ResourceNames;
* @author Franz-Josef Elmer
*/
@Component(value = ResourceNames.GENERIC_PLUGIN_SERVICE)
-public class GenericClientService extends AbstractClientService implements IGenericClientService
+public class GenericClientService extends AbstractClientService implements IGenericClientService, IEntityImportService
{
@Resource(name = ResourceNames.GENERIC_PLUGIN_SERVER)
diff --git a/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/ExecuteOperationsTest.java b/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/ExecuteOperationsTest.java
index 73e604568179991bd24386f1b0de10c43fe504ba..034b5a18038fab6423f3da7256452c031893a5ad 100644
--- a/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/ExecuteOperationsTest.java
+++ b/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/ExecuteOperationsTest.java
@@ -23,19 +23,13 @@ import static org.testng.Assert.assertEquals;
import static org.testng.Assert.assertNotNull;
import static org.testng.Assert.assertNull;
-import java.io.File;
-import java.io.IOException;
import java.lang.reflect.Method;
import java.util.Arrays;
-import java.util.Comparator;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
-import java.util.regex.Matcher;
-import java.util.regex.Pattern;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.time.DateUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.transaction.PlatformTransactionManager;
@@ -92,6 +86,8 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.update.SpaceUpdate;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.update.UpdateSpacesOperation;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.update.UpdateSpacesOperationResult;
+import ch.ethz.sis.openbis.systemtest.asapi.v3.util.EmailUtil;
+import ch.ethz.sis.openbis.systemtest.asapi.v3.util.EmailUtil.Email;
import ch.systemsx.cisd.common.action.IDelegatedAction;
import ch.systemsx.cisd.common.test.AssertionUtil;
import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO;
@@ -106,10 +102,6 @@ public class ExecuteOperationsTest extends AbstractOperationExecutionTest
private static final int SECONDS_PER_DAY = SECONDS_PER_HOUR * 24;
- private static final String EMAIL_DIR = "targets/email";
-
- private static final String EMAIL_PATTERN = "Date: (.*)\nFrom: (.*)\nTo: (.*)\nSubject: (.*)\nContent:\n(.*)";
-
private static final String EMAIL_FROM = "application_server@localhost";
private static final String EMAIL_TO_1 = "test1@email.com";
@@ -412,7 +404,7 @@ public class ExecuteOperationsTest extends AbstractOperationExecutionTest
assertEquals(historyEntry.getPropertyName(), "COMMENT");
assertEquals(historyEntry.getPropertyValue(), "created");
}
-
+
@Test
public void testUpdateDataSetAndRelatedSample()
{
@@ -842,7 +834,7 @@ public class ExecuteOperationsTest extends AbstractOperationExecutionTest
CreateSpacesOperationResult result = (CreateSpacesOperationResult) results.getResults().get(0);
assertEquals(result.getObjectIds(), Arrays.asList(new SpacePermId(creation.getCode())));
- Email email = findLatestEmail();
+ Email email = EmailUtil.findLatestEmail();
assertEquals(email.from, EMAIL_FROM);
assertEquals(email.to, EMAIL_TO_1 + ", " + EMAIL_TO_2);
assertEquals(email.subject, String.format("Operation execution %s finished", options.getExecutionId()));
@@ -872,7 +864,7 @@ public class ExecuteOperationsTest extends AbstractOperationExecutionTest
}
}, "Code cannot be empty");
- Email email = findLatestEmail();
+ Email email = EmailUtil.findLatestEmail();
assertEquals(email.from, EMAIL_FROM);
assertEquals(email.to, EMAIL_TO_1 + ", " + EMAIL_TO_2);
assertEquals(email.subject, String.format("Operation execution %s failed", options.getExecutionId()));
@@ -897,61 +889,6 @@ public class ExecuteOperationsTest extends AbstractOperationExecutionTest
return getExecution(sessionToken, options.getExecutionId(), fullOperationExecutionFetchOptions());
}
- private Email findLatestEmail()
- {
- File emailDir = new File(EMAIL_DIR);
-
- if (emailDir.exists())
- {
- File[] emails = emailDir.listFiles();
-
- if (emails != null && emails.length > 0)
- {
- Arrays.sort(emails, new Comparator<File>()
- {
- @Override
- public int compare(File f1, File f2)
- {
- return -f1.getName().compareTo(f2.getName());
- }
- });
-
- File latestEmail = emails[0];
- try
- {
- String latestEmailContent = FileUtils.readFileToString(latestEmail);
- Pattern pattern = Pattern.compile(EMAIL_PATTERN, Pattern.DOTALL);
-
- Matcher m = pattern.matcher(latestEmailContent);
- if (m.find())
- {
- Email email = new Email();
- email.from = m.group(2);
- email.to = m.group(3);
- email.subject = m.group(4);
- email.content = m.group(5);
- return email;
- } else
- {
- throw new RuntimeException("Latest email content does not match the expected email pattern. The latest email content was:\n"
- + latestEmailContent + "\nThe expected email pattern was:\n" + EMAIL_PATTERN);
- }
-
- } catch (IOException e)
- {
- throw new RuntimeException("Could not read the latest email " + latestEmail.getAbsolutePath(), e);
- }
-
- } else
- {
- throw new RuntimeException("No emails found in " + emailDir.getAbsolutePath() + " directory");
- }
- } else
- {
- throw new RuntimeException("Email directory " + emailDir.getAbsolutePath() + " does not exist");
- }
- }
-
private Set<String> getAllSpaceCodes()
{
// do it in a separate transaction to get only spaces that won't be rolled back
@@ -992,15 +929,4 @@ public class ExecuteOperationsTest extends AbstractOperationExecutionTest
}
}
- private class Email
- {
- private String from;
-
- private String to;
-
- private String subject;
-
- private String content;
- }
-
}
\ No newline at end of file
diff --git a/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/util/EmailUtil.java b/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/util/EmailUtil.java
new file mode 100644
index 0000000000000000000000000000000000000000..ccf9b78fbefe860c61a3fcbb19a3bbac84060dd1
--- /dev/null
+++ b/openbis/sourceTest/java/ch/ethz/sis/openbis/systemtest/asapi/v3/util/EmailUtil.java
@@ -0,0 +1,115 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.systemtest.asapi.v3.util;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Comparator;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.lang.builder.ReflectionToStringBuilder;
+
+/**
+ * @author pkupczyk
+ */
+public class EmailUtil
+{
+
+ private static final String EMAIL_DIR = "targets/email";
+
+ private static final String EMAIL_PATTERN = "Date: (.*)\nFrom: (.*)\nTo: (.*)\nSubject: (.*)\nContent:\n(.*)";
+
+ public static Email findLatestEmail()
+ {
+ File emailDir = new File(EMAIL_DIR);
+
+ if (emailDir.exists())
+ {
+ File[] emails = emailDir.listFiles();
+
+ if (emails != null && emails.length > 0)
+ {
+ Arrays.sort(emails, new Comparator<File>()
+ {
+ @Override
+ public int compare(File f1, File f2)
+ {
+ return -f1.getName().compareTo(f2.getName());
+ }
+ });
+
+ File latestEmail = emails[0];
+ try
+ {
+ String latestEmailContent = FileUtils.readFileToString(latestEmail);
+ Pattern pattern = Pattern.compile(EMAIL_PATTERN, Pattern.DOTALL);
+
+ Matcher m = pattern.matcher(latestEmailContent);
+ if (m.find())
+ {
+ Email email = new Email();
+ email.timestamp = latestEmail.lastModified();
+ email.from = m.group(2);
+ email.to = m.group(3);
+ email.subject = m.group(4);
+ email.content = m.group(5);
+ return email;
+ } else
+ {
+ throw new RuntimeException("Latest email content does not match the expected email pattern. The latest email content was:\n"
+ + latestEmailContent + "\nThe expected email pattern was:\n" + EMAIL_PATTERN);
+ }
+
+ } catch (IOException e)
+ {
+ throw new RuntimeException("Could not read the latest email " + latestEmail.getAbsolutePath(), e);
+ }
+
+ } else
+ {
+ return null;
+ }
+ } else
+ {
+ throw new RuntimeException("Email directory " + emailDir.getAbsolutePath() + " does not exist");
+ }
+ }
+
+ public static class Email
+ {
+
+ public long timestamp;
+
+ public String from;
+
+ public String to;
+
+ public String subject;
+
+ public String content;
+
+ @Override
+ public String toString()
+ {
+ return new ReflectionToStringBuilder(this).toString();
+ }
+ }
+
+}
diff --git a/openbis_api/source/java/ch/ethz/sis/openbis/generic/asapi/v3/plugin/service/IImportService.java b/openbis_api/source/java/ch/ethz/sis/openbis/generic/asapi/v3/plugin/service/IImportService.java
new file mode 100644
index 0000000000000000000000000000000000000000..b1c439a3df551a295ce1713c41f8793a692f002e
--- /dev/null
+++ b/openbis_api/source/java/ch/ethz/sis/openbis/generic/asapi/v3/plugin/service/IImportService.java
@@ -0,0 +1,48 @@
+/*
+ * Copyright 2018 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.ethz.sis.openbis.generic.asapi.v3.plugin.service;
+
+/**
+ * @author pkupczyk
+ */
+public interface IImportService
+{
+
+ public String createExperiments(String sessionToken, String uploadKey, String experimentTypeCode, boolean async, String userEmail);
+
+ public String updateExperiments(String sessionToken, String uploadKey, String experimentTypeCode, boolean async, String userEmail);
+
+ public String createSamples(String sessionToken, String uploadKey, String sampleTypeCode, String defaultSpaceIdentifier,
+ String spaceIdentifierOverride, String experimentIdentifierOverride, boolean updateExisting, boolean async, String userEmail);
+
+ public String updateSamples(String sessionToken, String uploadKey, String sampleTypeCode, String defaultSpaceIdentifier,
+ String spaceIdentifierOverride, String experimentIdentifierOverride, boolean async, String userEmail);
+
+ public String updateDataSets(String sessionToken, String uploadKey, String dataSetTypeCode, boolean async, String userEmail);
+
+ public String createMaterials(String sessionToken, String uploadKey, String materialTypeCode, boolean updateExisting, boolean async,
+ String userEmail);
+
+ public String updateMaterials(String sessionToken, String uploadKey, String materialTypeCode, boolean ignoreUnregistered, boolean async,
+ String userEmail);
+
+ public String generalImport(String sessionToken, String uploadKey, String defaultSpaceIdentifier, boolean updateExisting, boolean async,
+ String userEmail);
+
+ public String customImport(String sessionToken, String uploadKey, String customImportCode, boolean async, String userEmail);
+
+}