LMS-1502 Import CINA bundle data sets, with sub-data sets referencing the...

LMS-1502 Import CINA bundle data sets, with sub-data sets referencing the original using hard links.

SVN: 16371

datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetHandlerRpc.java

@@ -41,6 +41,15 @@ public interface IDataSetHandlerRpc extends IDataSetHandler
     public List<DataSetInformation> handleDataSet(final File dataSet,
             final DataSetInformation template);
 
+    /**
+     * Create a (hard) link to an file or folder within an existing data set and handle that.
+     * 
+     * @param dataSetComponent A file within a data set
+     * @param template A template containing information for the data sets to register
+     */
+    public List<DataSetInformation> linkAndHandleDataSet(File dataSetComponent,
+            DataSetInformation template);
+
     /**
      * Returns the session context for the current user.
      */

datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetExecutor.java

@@ -32,8 +32,10 @@ import org.apache.log4j.Logger;
 
 import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
 import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.filesystem.FastRecursiveHardLinkMaker;
 import ch.systemsx.cisd.common.filesystem.FileOperations;
 import ch.systemsx.cisd.common.filesystem.IFileOperations;
+import ch.systemsx.cisd.common.filesystem.IImmutableCopier;
 import ch.systemsx.cisd.common.io.ConcatenatedFileOutputStreamWriter;
 import ch.systemsx.cisd.common.mail.IMailClient;
 import ch.systemsx.cisd.common.utilities.IDelegatedActionWithResult;
@@ -92,6 +94,8 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
 
     private final IDataSetHandler handler;
 
+    private final IImmutableCopier copier;
+
     private DataSetInformation override;
 
     PutDataSetExecutor(PutDataSetService service, IETLServerPlugin plugin, String sessionToken,
@@ -102,6 +106,7 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
         this.sessionToken = sessionToken;
         this.newDataSet = newDataSet;
         this.inputStream = inputStream;
+        this.copier = FastRecursiveHardLinkMaker.tryCreate();
         this.dataSetDir = new File(service.getIncomingDir(), newDataSet.getDataSetFolderName());
         if (dataSetDir.exists())
         {
@@ -166,7 +171,7 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
             override = newOverride;
         }
 
-        RegistrationHelper helper = new RegistrationHelper(service, plugin, dataSetDir);
+        RegistrationHelper helper = new RegistrationHelper(service, plugin, dataSet);
         helper.prepare();
         if (helper.hasDataSetBeenIdentified())
         {
@@ -183,11 +188,48 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
         return Collections.singletonList(helper.getDataSetInformation());
     }
 
+    public List<DataSetInformation> linkAndHandleDataSet(File dataSetComponent,
+            DataSetInformation newOverride)
+    {
+        File incomingDir = service.getIncomingDir();
+        // Make a hard link to the file within the data set
+        boolean success;
+        success = copier.copyImmutably(dataSetComponent, incomingDir, null);
+        if (success == false)
+        {
+            throw new EnvironmentFailureException("Couldn't create a hard-link copy of '"
+                    + dataSetComponent.getAbsolutePath() + "' in folder '"
+                    + service.getIncomingDir().getAbsolutePath() + "'.");
+        }
+
+        File linkedFile = new File(incomingDir, dataSetComponent.getName());
+
+        // Register the component of the data set
+        try
+        {
+            return handleDataSet(linkedFile, newOverride);
+        } finally
+        {
+            deleteDir(linkedFile);
+        }
+    }
+
     public SessionContextDTO getSessionContext()
     {
         return getOpenBisService().tryGetSession(sessionToken);
     }
 
+    public File getFileForExternalData(ExternalData externalData)
+    {
+        File dataSetFile = new File(service.getStoreRootDirectory(), externalData.getLocation());
+        return DefaultStorageProcessor.getOriginalDirectory(dataSetFile);
+    }
+
+    public String getDataStoreCode()
+    {
+        return service.getDataStoreCode();
+    }
+
     private void createDefaultOverride()
     {
         override = new DataSetInformation();
@@ -251,10 +293,15 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
     }
 
     private void deleteDataSetDir()
+    {
+        deleteDir(dataSetDir);
+    }
+
+    private void deleteDir(File dirToDelete)
     {
         try
         {
-            FileUtils.deleteDirectory(dataSetDir);
+            FileUtils.deleteDirectory(dirToDelete);
         } catch (IOException ex)
         {
             getOperationLog().error("Could not delete data set directory " + dataSetDir, ex);
@@ -529,7 +576,7 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
 
             // Override / extend information extracted with our override
             dataSetInfo.setExperimentIdentifier(override.getExperimentIdentifier());
-            SampleIdentifier sampleIdOrNull = dataSetInfo.getSampleIdentifier();
+            SampleIdentifier sampleIdOrNull = override.getSampleIdentifier();
             if (sampleIdOrNull != null)
             {
                 dataSetInfo.setSampleCode(sampleIdOrNull.getSampleCode());
@@ -552,10 +599,4 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
             return dataSetInfo;
         }
     }
-
-    public File getFileForExternalData(ExternalData externalData)
-    {
-        File dataSetFile = new File(service.getStoreRootDirectory(), externalData.getLocation());
-        return DefaultStorageProcessor.getOriginalDirectory(dataSetFile);
-    }
 }

rtd_cina/source/java/ch/systemsx/cisd/cina/dss/bundle/BundleDataSetHelperRpc.java

@@ -17,6 +17,7 @@
 package ch.systemsx.cisd.cina.dss.bundle;
 
 import java.io.File;
+import java.util.Collections;
 
 import ch.systemsx.cisd.cina.shared.metadata.BundleMetadataExtractor;
 import ch.systemsx.cisd.cina.shared.metadata.ImageMetadataExtractor;
@@ -81,7 +82,6 @@ class BundleDataSetHelperRpc extends BundleDataSetHelper
 
         BundleMetadataExtractor bundleMetadata = new BundleMetadataExtractor(bundle);
         bundleMetadata.prepare();
-        System.out.println(bundleMetadata.getMetadataExtractors());
         for (ReplicaMetadataExtractor replicaMetadata : bundleMetadata.getMetadataExtractors())
         {
             handleDerivedDataSets(replicaMetadata);
@@ -145,18 +145,27 @@ class BundleDataSetHelperRpc extends BundleDataSetHelper
     {
         // Register a data set for the image
         File imageDataSet = imageMetadata.getFolder();
-        IDataSetHandler delegator = getDelegator();
-        if (delegator instanceof IDataSetHandlerRpc)
-        {
-            DataSetInformation template = new DataSetInformation();
-            template.setSampleCode(sampleId.getSampleCode());
-            template.setSpaceCode(sampleId.getSpaceLevel().getSpaceCode());
-            template.setInstanceCode(sampleId.getSpaceLevel().getDatabaseInstanceCode());
-            // ((IDataSetHandlerRpc) delegator).handleDataSet(imageDataSet, template);
-        } else
-        {
-            delegator.handleDataSet(imageDataSet);
-        }
+        DataSetInformation imageDataSetInfo = new DataSetInformation();
+        imageDataSetInfo.setSampleCode(sampleId.getSampleCode());
+        imageDataSetInfo.setSpaceCode(sampleId.getSpaceLevel().getSpaceCode());
+        imageDataSetInfo.setInstanceCode(sampleId.getSpaceLevel().getDatabaseInstanceCode());
+        imageDataSetInfo.setDataSetType(globalState.getImageDataSetType().getDataSetType());
+        imageDataSetInfo.setParentDataSetCodes(Collections.singletonList(getBigDataSetInformation()
+                .getDataSetCode()));
+
+        // NewExternalData data = new NewExternalData();
+        // data.setExtractableData(imageDataSetInfo.getExtractableData());
+        // data.setLocatorType(new LocatorType(LocatorType.DEFAULT_LOCATOR_TYPE_CODE));
+        // data.setDataSetType(imageDataSetInfo.getDataSetType());
+        // data.setFileFormatType(new FileFormatType(FileFormatType.DEFAULT_FILE_FORMAT_TYPE_CODE));
+        // data.setMeasured(true);
+        // data.setDataStoreCode(delegatorRpc.getDataStoreCode());
+        // data.setComplete(BooleanOrUnknown.T);
+        // data.setLocation(delegatorRpc.getStoreRelativePath(imageDataSet));
+        // data.setStorageFormat(StorageFormat.PROPRIETARY);
+
+        // getOpenbisService().registerDataSet(imageDataSetInfo, data);
+        delegatorRpc.linkAndHandleDataSet(imageDataSet, imageDataSetInfo);
     }
 
     /**

rtd_cina/source/java/ch/systemsx/cisd/cina/shared/metadata/BundleMetadataExtractor.java

@@ -89,8 +89,6 @@ public class BundleMetadataExtractor
             return;
         }
 
-        System.out.println(file);
-
         if (false == ReplicaMetadataExtractor.doesFolderContainReplicaMetadata(file))
         {
             return;

rtd_cina/sourceTest/java/ch/systemsx/cisd/cina/dss/bundle/CinaBundleDataSetHandlerTest.java

@@ -91,11 +91,11 @@ public class CinaBundleDataSetHandlerTest extends AbstractFileSystemTestCase
                 {
                     final DataSetInformation dataSetInformation = new DataSetInformation();
                     dataSetInformation.setDataSetCode("Derived");
-                    dataSetInformation.setSampleCode("2");
+                    dataSetInformation.setSampleCode("" + uniqueId++);
                     dataSetInformation.setSpaceCode("Space");
                     dataSetInformation.setInstanceCode("Test");
 
-                    allowing(delegator).handleDataSet(with(new File(path)),
+                    allowing(delegator).linkAndHandleDataSet(with(new File(path)),
                             with(any(DataSetInformation.class)));
                     will(returnValue(Collections.singletonList(dataSetInformation)));
                 }