From ea831ec27385fdeef3a226e53d3353b52eb35dce Mon Sep 17 00:00:00 2001
From: cramakri <cramakri>
Date: Wed, 9 Jun 2010 15:06:33 +0000
Subject: [PATCH] LMS-1502 Import CINA bundle data sets, with sub-data sets
referencing the original using hard links.
SVN: 16371
---
.../cisd/etlserver/IDataSetHandlerRpc.java | 9 +++
.../etlserver/api/v1/PutDataSetExecutor.java | 59 ++++++++++++++++---
.../dss/bundle/BundleDataSetHelperRpc.java | 35 +++++++----
.../metadata/BundleMetadataExtractor.java | 2 -
.../bundle/CinaBundleDataSetHandlerTest.java | 4 +-
5 files changed, 83 insertions(+), 26 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetHandlerRpc.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetHandlerRpc.java
index 485f03b0719..d3405ad9731 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetHandlerRpc.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetHandlerRpc.java
@@ -41,6 +41,15 @@ public interface IDataSetHandlerRpc extends IDataSetHandler
public List<DataSetInformation> handleDataSet(final File dataSet,
final DataSetInformation template);
+ /**
+ * Create a (hard) link to an file or folder within an existing data set and handle that.
+ *
+ * @param dataSetComponent A file within a data set
+ * @param template A template containing information for the data sets to register
+ */
+ public List<DataSetInformation> linkAndHandleDataSet(File dataSetComponent,
+ DataSetInformation template);
+
/**
* Returns the session context for the current user.
*/
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetExecutor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetExecutor.java
index 783ce829134..9552d0c056e 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetExecutor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetExecutor.java
@@ -32,8 +32,10 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
+import ch.systemsx.cisd.common.filesystem.FastRecursiveHardLinkMaker;
import ch.systemsx.cisd.common.filesystem.FileOperations;
import ch.systemsx.cisd.common.filesystem.IFileOperations;
+import ch.systemsx.cisd.common.filesystem.IImmutableCopier;
import ch.systemsx.cisd.common.io.ConcatenatedFileOutputStreamWriter;
import ch.systemsx.cisd.common.mail.IMailClient;
import ch.systemsx.cisd.common.utilities.IDelegatedActionWithResult;
@@ -92,6 +94,8 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
private final IDataSetHandler handler;
+ private final IImmutableCopier copier;
+
private DataSetInformation override;
PutDataSetExecutor(PutDataSetService service, IETLServerPlugin plugin, String sessionToken,
@@ -102,6 +106,7 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
this.sessionToken = sessionToken;
this.newDataSet = newDataSet;
this.inputStream = inputStream;
+ this.copier = FastRecursiveHardLinkMaker.tryCreate();
this.dataSetDir = new File(service.getIncomingDir(), newDataSet.getDataSetFolderName());
if (dataSetDir.exists())
{
@@ -166,7 +171,7 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
override = newOverride;
}
- RegistrationHelper helper = new RegistrationHelper(service, plugin, dataSetDir);
+ RegistrationHelper helper = new RegistrationHelper(service, plugin, dataSet);
helper.prepare();
if (helper.hasDataSetBeenIdentified())
{
@@ -183,11 +188,48 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
return Collections.singletonList(helper.getDataSetInformation());
}
+ public List<DataSetInformation> linkAndHandleDataSet(File dataSetComponent,
+ DataSetInformation newOverride)
+ {
+ File incomingDir = service.getIncomingDir();
+ // Make a hard link to the file within the data set
+ boolean success;
+ success = copier.copyImmutably(dataSetComponent, incomingDir, null);
+ if (success == false)
+ {
+ throw new EnvironmentFailureException("Couldn't create a hard-link copy of '"
+ + dataSetComponent.getAbsolutePath() + "' in folder '"
+ + service.getIncomingDir().getAbsolutePath() + "'.");
+ }
+
+ File linkedFile = new File(incomingDir, dataSetComponent.getName());
+
+ // Register the component of the data set
+ try
+ {
+ return handleDataSet(linkedFile, newOverride);
+ } finally
+ {
+ deleteDir(linkedFile);
+ }
+ }
+
public SessionContextDTO getSessionContext()
{
return getOpenBisService().tryGetSession(sessionToken);
}
+ public File getFileForExternalData(ExternalData externalData)
+ {
+ File dataSetFile = new File(service.getStoreRootDirectory(), externalData.getLocation());
+ return DefaultStorageProcessor.getOriginalDirectory(dataSetFile);
+ }
+
+ public String getDataStoreCode()
+ {
+ return service.getDataStoreCode();
+ }
+
private void createDefaultOverride()
{
override = new DataSetInformation();
@@ -251,10 +293,15 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
}
private void deleteDataSetDir()
+ {
+ deleteDir(dataSetDir);
+ }
+
+ private void deleteDir(File dirToDelete)
{
try
{
- FileUtils.deleteDirectory(dataSetDir);
+ FileUtils.deleteDirectory(dirToDelete);
} catch (IOException ex)
{
getOperationLog().error("Could not delete data set directory " + dataSetDir, ex);
@@ -529,7 +576,7 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
// Override / extend information extracted with our override
dataSetInfo.setExperimentIdentifier(override.getExperimentIdentifier());
- SampleIdentifier sampleIdOrNull = dataSetInfo.getSampleIdentifier();
+ SampleIdentifier sampleIdOrNull = override.getSampleIdentifier();
if (sampleIdOrNull != null)
{
dataSetInfo.setSampleCode(sampleIdOrNull.getSampleCode());
@@ -552,10 +599,4 @@ class PutDataSetExecutor implements IDataSetHandlerRpc
return dataSetInfo;
}
}
-
- public File getFileForExternalData(ExternalData externalData)
- {
- File dataSetFile = new File(service.getStoreRootDirectory(), externalData.getLocation());
- return DefaultStorageProcessor.getOriginalDirectory(dataSetFile);
- }
}
diff --git a/rtd_cina/source/java/ch/systemsx/cisd/cina/dss/bundle/BundleDataSetHelperRpc.java b/rtd_cina/source/java/ch/systemsx/cisd/cina/dss/bundle/BundleDataSetHelperRpc.java
index d3ded74986a..455b4ef66a3 100644
--- a/rtd_cina/source/java/ch/systemsx/cisd/cina/dss/bundle/BundleDataSetHelperRpc.java
+++ b/rtd_cina/source/java/ch/systemsx/cisd/cina/dss/bundle/BundleDataSetHelperRpc.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.cina.dss.bundle;
import java.io.File;
+import java.util.Collections;
import ch.systemsx.cisd.cina.shared.metadata.BundleMetadataExtractor;
import ch.systemsx.cisd.cina.shared.metadata.ImageMetadataExtractor;
@@ -81,7 +82,6 @@ class BundleDataSetHelperRpc extends BundleDataSetHelper
BundleMetadataExtractor bundleMetadata = new BundleMetadataExtractor(bundle);
bundleMetadata.prepare();
- System.out.println(bundleMetadata.getMetadataExtractors());
for (ReplicaMetadataExtractor replicaMetadata : bundleMetadata.getMetadataExtractors())
{
handleDerivedDataSets(replicaMetadata);
@@ -145,18 +145,27 @@ class BundleDataSetHelperRpc extends BundleDataSetHelper
{
// Register a data set for the image
File imageDataSet = imageMetadata.getFolder();
- IDataSetHandler delegator = getDelegator();
- if (delegator instanceof IDataSetHandlerRpc)
- {
- DataSetInformation template = new DataSetInformation();
- template.setSampleCode(sampleId.getSampleCode());
- template.setSpaceCode(sampleId.getSpaceLevel().getSpaceCode());
- template.setInstanceCode(sampleId.getSpaceLevel().getDatabaseInstanceCode());
- // ((IDataSetHandlerRpc) delegator).handleDataSet(imageDataSet, template);
- } else
- {
- delegator.handleDataSet(imageDataSet);
- }
+ DataSetInformation imageDataSetInfo = new DataSetInformation();
+ imageDataSetInfo.setSampleCode(sampleId.getSampleCode());
+ imageDataSetInfo.setSpaceCode(sampleId.getSpaceLevel().getSpaceCode());
+ imageDataSetInfo.setInstanceCode(sampleId.getSpaceLevel().getDatabaseInstanceCode());
+ imageDataSetInfo.setDataSetType(globalState.getImageDataSetType().getDataSetType());
+ imageDataSetInfo.setParentDataSetCodes(Collections.singletonList(getBigDataSetInformation()
+ .getDataSetCode()));
+
+ // NewExternalData data = new NewExternalData();
+ // data.setExtractableData(imageDataSetInfo.getExtractableData());
+ // data.setLocatorType(new LocatorType(LocatorType.DEFAULT_LOCATOR_TYPE_CODE));
+ // data.setDataSetType(imageDataSetInfo.getDataSetType());
+ // data.setFileFormatType(new FileFormatType(FileFormatType.DEFAULT_FILE_FORMAT_TYPE_CODE));
+ // data.setMeasured(true);
+ // data.setDataStoreCode(delegatorRpc.getDataStoreCode());
+ // data.setComplete(BooleanOrUnknown.T);
+ // data.setLocation(delegatorRpc.getStoreRelativePath(imageDataSet));
+ // data.setStorageFormat(StorageFormat.PROPRIETARY);
+
+ // getOpenbisService().registerDataSet(imageDataSetInfo, data);
+ delegatorRpc.linkAndHandleDataSet(imageDataSet, imageDataSetInfo);
}
/**
diff --git a/rtd_cina/source/java/ch/systemsx/cisd/cina/shared/metadata/BundleMetadataExtractor.java b/rtd_cina/source/java/ch/systemsx/cisd/cina/shared/metadata/BundleMetadataExtractor.java
index 1a27426c7b4..ad39ada41b0 100644
--- a/rtd_cina/source/java/ch/systemsx/cisd/cina/shared/metadata/BundleMetadataExtractor.java
+++ b/rtd_cina/source/java/ch/systemsx/cisd/cina/shared/metadata/BundleMetadataExtractor.java
@@ -89,8 +89,6 @@ public class BundleMetadataExtractor
return;
}
- System.out.println(file);
-
if (false == ReplicaMetadataExtractor.doesFolderContainReplicaMetadata(file))
{
return;
diff --git a/rtd_cina/sourceTest/java/ch/systemsx/cisd/cina/dss/bundle/CinaBundleDataSetHandlerTest.java b/rtd_cina/sourceTest/java/ch/systemsx/cisd/cina/dss/bundle/CinaBundleDataSetHandlerTest.java
index e12349f6bac..2f27648753e 100644
--- a/rtd_cina/sourceTest/java/ch/systemsx/cisd/cina/dss/bundle/CinaBundleDataSetHandlerTest.java
+++ b/rtd_cina/sourceTest/java/ch/systemsx/cisd/cina/dss/bundle/CinaBundleDataSetHandlerTest.java
@@ -91,11 +91,11 @@ public class CinaBundleDataSetHandlerTest extends AbstractFileSystemTestCase
{
final DataSetInformation dataSetInformation = new DataSetInformation();
dataSetInformation.setDataSetCode("Derived");
- dataSetInformation.setSampleCode("2");
+ dataSetInformation.setSampleCode("" + uniqueId++);
dataSetInformation.setSpaceCode("Space");
dataSetInformation.setInstanceCode("Test");
- allowing(delegator).handleDataSet(with(new File(path)),
+ allowing(delegator).linkAndHandleDataSet(with(new File(path)),
with(any(DataSetInformation.class)));
will(returnValue(Collections.singletonList(dataSetInformation)));
}
--
GitLab