SSDM-7275 : property translator

microscopy-migration-tool/readme.txt

-The tool requires a tags.txt file on the root.
+** The tool requires:
+
+1. DSS service.properties configured with:
+
+dss-rpc.put-default = eln-lims-dropbox
+dss-rpc.put.ATTACHMENT = eln-lims-dropbox
+
+2. A tags.txt file on the root to migrate the tags
 
 The information can be retrieved from the database using the next SQL:
 
@@ -8,4 +15,13 @@ LEFT JOIN metaprojects m ON (m.id = ma.mepr_id)
 LEFT JOIN experiments_all ea ON (ea.id = ma.expe_id);
 
 It follows the next format, including quotes, one tag per line:
-"TagName"\t"permId"
\ No newline at end of file
+"TagName"\t"permId"
+
+** The tool produces:
+
+1. Output of the process.
+
+2. A file called openbis_audit_data_update.sql.
+
+This file is a sql script that needs to be executed.
+It migrates the audit data from the translated experiments to samples.
\ No newline at end of file

microscopy-migration-tool/src/ethz/ch/Main.java

@@ -13,6 +13,9 @@ import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.search.ExperimentSear
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.Project;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.id.ProjectIdentifier;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.project.search.ProjectSearchCriteria;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.search.SampleSearchCriteria;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.Space;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.fetchoptions.SpaceFetchOptions;
 import ch.ethz.sis.openbis.generic.asapi.v3.dto.space.search.SpaceSearchCriteria;
@@ -22,6 +25,8 @@ import ethz.ch.dataset.DatasetCreationHelper;
 import ethz.ch.experiment.Experiment2Sample;
 import ethz.ch.experiment.Experiment2SampleTranslator;
 import ethz.ch.experiment.ExperimentType2SampleType;
+import ethz.ch.property.Property2Sample;
+import ethz.ch.property.Property2SampleTranslator;
 import ethz.ch.property.PropertyType2SampleType;
 import ethz.ch.ssl.SslCertificateHelper;
 import ethz.ch.tag.Tag2SampleTranslator;
@@ -191,10 +196,6 @@ public class Main
             }
         }
         
-        //
-        //
-        //
-        
         System.out.println("4. Translate Properties to Samples");
 
         PropertyType2SampleType FACS_ARIA_TUBE =    new PropertyType2SampleType(        "FACS_ARIA_TUBE",   "FACS_ARIA_SPECIMEN",   "FACS_ARIA_SPECIMEN",   "$NAME");
@@ -213,5 +214,26 @@ public class Main
                 MOFLO_XDP_TUBE,
                 SE3_TUBE);
         
+        for(PropertyType2SampleType propertyMigrationConfigs:propertiesMigrationConfigs) {
+            if(COMMIT_CHANGES_TO_OPENBIS && !MasterdataHelper.doSampleTypeExist(sessionToken, v3, propertyMigrationConfigs.getNewSampleTypeCode())) {
+                MasterdataHelper.createDefaultSampleType(sessionToken, v3, propertyMigrationConfigs.getNewSampleTypeCode());
+                System.out.println(propertyMigrationConfigs.getNewSampleTypeCode() + " Sample Type installed.");
+            } else {
+                System.out.println(propertyMigrationConfigs.getNewSampleTypeCode() + " Sample Type installation skipped.");
+            }
+        }
+        
+        for(PropertyType2SampleType config:propertiesMigrationConfigs) {
+            int total = 0;
+            SampleSearchCriteria sampleSearchCriteria = new SampleSearchCriteria();
+            sampleSearchCriteria.withType().withCode().thatEquals(config.getOldSampleTypeCode());
+            SearchResult<Sample> samples = v3.searchSamples(sessionToken, sampleSearchCriteria, new SampleFetchOptions());
+            for(Sample sample:samples.getObjects()) {
+                Property2Sample toMigrate = new Property2Sample(config, sample.getPermId());
+                Property2SampleTranslator.translate(sessionToken, v3, v3dss, toMigrate, COMMIT_CHANGES_TO_OPENBIS);
+                total++;
+                System.out.println("[DONE] " + config.getOldSampleTypeCode() + "\t" + total + "/" + samples.getTotalCount());
+            }
+        }
     }
 }

microscopy-migration-tool/src/ethz/ch/MasterdataHelper.java

@@ -255,6 +255,11 @@ public class MasterdataHelper
     // Experiment Type to Sample Type translator
     //
     
+    public static void createDefaultSampleType(String sessionToken, IApplicationServerApi v3, String sampleTypeCode) {
+        SampleTypeCreation stc = getDefaultSampleType(sampleTypeCode, "N/A", true);
+        createSampleTypeIfMissing(sessionToken, v3, stc);
+    }
+    
     public static void createSampleTypesFromExperimentTypes(String sessionToken, IApplicationServerApi v3, List<String> experimentTypeIds) {
         List<EntityTypePermId> ids = new ArrayList<EntityTypePermId>();
         for(String experimentTypeId:experimentTypeIds) {
@@ -270,10 +275,13 @@ public class MasterdataHelper
         }
     }
     
-    public static SampleTypeCreation getSampleTypeFromExperimentType(ExperimentType experimentType) {
+    public static SampleTypeCreation getDefaultSampleType(String sampleTypeCode, String sampleTypeDescription, boolean autoGeneratedCode) {
         SampleTypeCreation creation = new SampleTypeCreation();
-        creation.setCode(experimentType.getCode());
-        creation.setDescription(experimentType.getDescription());
+        creation.setCode(sampleTypeCode);
+        creation.setDescription(sampleTypeDescription);
+        creation.setAutoGeneratedCode(autoGeneratedCode);
+        creation.setGeneratedCodePrefix(sampleTypeCode + ".");
+        
         List<PropertyAssignmentCreation> propertyAssignmentCreations = new ArrayList<PropertyAssignmentCreation>();
         
         PropertyAssignmentCreation NAME = new PropertyAssignmentCreation();
@@ -289,14 +297,21 @@ public class MasterdataHelper
         propertyAssignmentCreations.add(NAME);
         propertyAssignmentCreations.add(ANNOTATIONS_STATE);
         
+        creation.setPropertyAssignments(propertyAssignmentCreations);
+        
+        return creation;
+    }
+    
+    public static SampleTypeCreation getSampleTypeFromExperimentType(ExperimentType experimentType) {
+        SampleTypeCreation creation = getDefaultSampleType(experimentType.getCode(), experimentType.getDescription(), false);
+        
         for(PropertyAssignment propertyAssignment:experimentType.getPropertyAssignments()) {
             PropertyAssignmentCreation pac = new PropertyAssignmentCreation();
             pac.setPropertyTypeId(propertyAssignment.getPropertyType().getPermId());
             pac.setShowInEditView(propertyAssignment.isShowInEditView());
             pac.setSection(propertyAssignment.getSection());
-            propertyAssignmentCreations.add(pac);
+            creation.getPropertyAssignments().add(pac);
         }
-        creation.setPropertyAssignments(propertyAssignmentCreations);
         
         return creation;
     }

microscopy-migration-tool/src/ethz/ch/MetadataHelper.java

@@ -61,12 +61,12 @@ public class MetadataHelper
         return experimentCreation;
     }
     
-    public static SampleCreation getOrganizationUnitCreation(ExperimentIdentifier experimentId, String name) {
+    public static SampleCreation getBasicSampleCreation(ExperimentIdentifier experimentIdentifier, String sampleTypeCode, String name) {
         SampleCreation sampleCreation = new SampleCreation();
-        sampleCreation.setTypeId(new EntityTypePermId("ORGANIZATION_UNIT"));
+        sampleCreation.setTypeId(new EntityTypePermId(sampleTypeCode));
         sampleCreation.setProperty("$NAME", name);
-        sampleCreation.setSpaceId(new SpacePermId(experimentId.getIdentifier().split("/")[1]));
-        sampleCreation.setExperimentId(experimentId);
+        sampleCreation.setSpaceId(new SpacePermId(experimentIdentifier.getIdentifier().split("/")[1]));
+        sampleCreation.setExperimentId(experimentIdentifier);
         return sampleCreation;
     }
     
@@ -134,17 +134,20 @@ public class MetadataHelper
     // Gets
     //
     
-    public static Sample getSample(IApplicationServerApi v3, String sessionToken, String sampleIdentifier) {
+    public static Sample getSample(String sessionToken, IApplicationServerApi v3, ISampleId sampleId) {
         SampleFetchOptions fo = new SampleFetchOptions();
         fo.withProperties();
         fo.withChildren();
         fo.withParents();
+        fo.withContainer();
+        fo.withSpace();
+        fo.withExperiment();
         fo.withDataSets().withType();
         fo.withRegistrator();
         fo.withModifier();
         fo.withAttachments();
-        Map<ISampleId, Sample> samples = v3.getSamples(sessionToken, Collections.singletonList(new SampleIdentifier(sampleIdentifier)), fo);
-        return samples.get(new SampleIdentifier(sampleIdentifier));
+        Map<ISampleId, Sample> samples = v3.getSamples(sessionToken, Collections.singletonList(sampleId), fo);
+        return samples.get(sampleId);
     }
     
     public static Experiment getExperiment(String sessionToken,  IApplicationServerApi v3, ExperimentPermId permId) {

microscopy-migration-tool/src/ethz/ch/experiment/Experiment2SampleTranslator.java

@@ -44,11 +44,11 @@ public class Experiment2SampleTranslator
         Experiment experiment = MetadataHelper.getExperiment(sessionToken, v3, toMigrate.getExperimentPermId());
         
         // Test mode, only with complete experiments
-        if(experiment.getAttachments().isEmpty() || 
-                experiment.getSamples().isEmpty() || 
-                experiment.getDataSets().isEmpty()) {
-            return;
-        }
+//        if(experiment.getAttachments().isEmpty() || 
+//                experiment.getSamples().isEmpty() || 
+//                experiment.getDataSets().isEmpty()) {
+//            return;
+//        }
         //
         
         SampleCreation sampleCreation = new SampleCreation();
@@ -78,7 +78,7 @@ public class Experiment2SampleTranslator
         // 2. Does the new sample exists? Create if not
         SampleIdentifier sampleIdentifier = new SampleIdentifier(experiment.getIdentifier().getIdentifier());
         SamplePermId samplePermId = null;
-        Sample sample = MetadataHelper.getSample(v3, sessionToken, sampleIdentifier.getIdentifier());
+        Sample sample = MetadataHelper.getSample(sessionToken, v3, sampleIdentifier);
         if(COMMIT_CHANGES_TO_OPENBIS && sample == null) {
             samplePermId = v3.createSamples(sessionToken, Collections.singletonList(sampleCreation)).get(0);
             System.out.println("Created Sample\t" + sampleIdentifier.getIdentifier());

microscopy-migration-tool/src/ethz/ch/property/Property2SampleTranslator.java

 package ethz.ch.property;
 
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
 import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.create.SampleCreation;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.ISampleId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.update.SampleUpdate;
 import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi;
+import ethz.ch.MetadataHelper;
 
 public class Property2SampleTranslator
 {
@@ -10,7 +23,65 @@ public class Property2SampleTranslator
             IDataStoreServerApi v3dss,
             Property2Sample toMigrate, 
             boolean COMMIT_CHANGES_TO_OPENBIS) throws Exception {
-            System.out.println("[START]\t" + toMigrate.getSamplePermId());
             
+        System.out.println("[START]\t" + toMigrate.getSamplePermId());
+        Sample sample = MetadataHelper.getSample(sessionToken, v3, toMigrate.getSamplePermId());
+        String propertyValue = sample.getProperties().get(toMigrate.getConfig().getOldPropertyCode());
+        
+        // If has not being assigned yet.
+        if(propertyValue == null || propertyValue.isEmpty()) {
+            System.out.println("[FINISH EMPTY]\t" + toMigrate.getSamplePermId());
+            return;
+        }
+        
+        ISampleId sampleId = null;
+        // 1. Try to find property sample.
+        sampleId = getSampleWithExperimentAndName(sessionToken, v3, toMigrate, sample.getExperiment().getIdentifier(), propertyValue);
+        
+        // 2. If property sample doesn't exist. Create a basic sample holding it and assign it to the experiment
+        if(sampleId == null) {
+            SampleCreation sampleCreation = new SampleCreation();
+            sampleCreation.setTypeId(new EntityTypePermId(toMigrate.getConfig().getNewSampleTypeCode()));
+            sampleCreation.getProperties().put(toMigrate.getConfig().getNewPropertyCode(), propertyValue);
+            
+            sampleCreation.setSpaceId(sample.getSpace().getPermId());
+            sampleCreation.setExperimentId(sample.getExperiment().getPermId());
+            
+            List<ISampleId> parentIds = new ArrayList<>();
+            parentIds.add(sample.getContainer().getPermId());
+            
+            sampleId = v3.createSamples(sessionToken, Collections.singletonList(sampleCreation)).iterator().next();
+            
+            // Add to cache to speed up calls
+            sampleCache.get(sample.getExperiment().getIdentifier()).put(propertyValue, sampleId);
+            
+            System.out.println("Sample Property\t" + propertyValue + "\t" + sample.getExperiment().getPermId());
+        }
+        
+        // 3. Assign the property sample as parent of the sample.
+        // 4. Delete old property to not creating it twice on retry.
+        SampleUpdate sampleUpdate = new SampleUpdate();
+        sampleUpdate.setSampleId(sample.getPermId());
+        sampleUpdate.setContainerId(null);
+        sampleUpdate.getParentIds().add(sampleId, sample.getContainer().getPermId());
+        sampleUpdate.setProperty(toMigrate.getConfig().getOldPropertyCode(), null);
+        v3.updateSamples(sessionToken, Collections.singletonList(sampleUpdate));
+        System.out.println("Set Parent\t" + propertyValue + "\t" + sample.getIdentifier().getIdentifier());
+        System.out.println("Set Parent\t" + sample.getContainer().getIdentifier() + "\t" + sample.getIdentifier().getIdentifier());
+        System.out.println("Delete Container" + sample.getContainer().getIdentifier() + "\t" + sample.getIdentifier().getIdentifier());
+        System.out.println("Delete property\t" + toMigrate.getConfig().getOldPropertyCode() + "\t" + sample.getIdentifier().getIdentifier());
+        System.out.println("[FINISH]\t" + toMigrate.getSamplePermId());
+    }
+    
+    private static Map<ExperimentIdentifier, Map<String, ISampleId>> sampleCache = new HashMap<>();
+    
+    public static ISampleId getSampleWithExperimentAndName(String sessionToken, IApplicationServerApi v3, Property2Sample toMigrate, ExperimentIdentifier experimentIdentifier, String name) {
+        Map<String, ISampleId> esPcache = sampleCache.get(experimentIdentifier);
+        if(esPcache == null) {
+            esPcache = new HashMap<>();
+            sampleCache.put(experimentIdentifier, esPcache);
+        }
+        ISampleId sampleForProperty = esPcache.get(name);
+        return sampleForProperty;
     }
 }

microscopy-migration-tool/src/ethz/ch/property/PropertyType2SampleType.java

@@ -2,23 +2,23 @@ package ethz.ch.property;
 
 public class PropertyType2SampleType
 {
-    private String oldSampleType;
+    private String oldSampleTypeCode;
     private String oldPropertyCode;
-    private String newSampleType;
+    private String newSampleTypeCode;
     private String newPropertyCode;
     
     public PropertyType2SampleType(String oldSampleType, String oldPropertyCode, String newSampleType, String newPropertyCode)
     {
         super();
-        this.oldSampleType = oldSampleType;
+        this.oldSampleTypeCode = oldSampleType;
         this.oldPropertyCode = oldPropertyCode;
-        this.newSampleType = newSampleType;
+        this.newSampleTypeCode = newSampleType;
         this.newPropertyCode = newPropertyCode;
     }
 
-    public String getOldSampleType()
+    public String getOldSampleTypeCode()
     {
-        return oldSampleType;
+        return oldSampleTypeCode;
     }
 
     public String getOldPropertyCode()
@@ -26,9 +26,9 @@ public class PropertyType2SampleType
         return oldPropertyCode;
     }
 
-    public String getNewSampleType()
+    public String getNewSampleTypeCode()
     {
-        return newSampleType;
+        return newSampleTypeCode;
     }
 
     public String getNewPropertyCode()

microscopy-migration-tool/src/ethz/ch/tag/Tag2SampleTranslator.java

@@ -76,7 +76,7 @@ public class Tag2SampleTranslator
                     if(ousByTagName.containsKey(tagRequired)) {
                         ousFound.add(ousByTagName.get(tagRequired).getPermId());
                     } else {
-                        ousMissing.add(MetadataHelper.getOrganizationUnitCreation(ouCollIdent, tagRequired));
+                        ousMissing.add(MetadataHelper.getBasicSampleCreation(ouCollIdent, "ORGANIZATION_UNIT", tagRequired));
                     }
                 }