From 04af97ce18e4886084f1f74b97a21f8e1d0c2dae Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Thu, 25 Nov 2010 08:02:52 +0000
Subject: [PATCH] SE-316 MsInjectionDataSetCopier renamed to DataSetCopier. New
feature: Optional creation of marker file.
SVN: 18892
---
rtd_phosphonetx/etc/service.properties | 7 ++-
...nDataSetCopier.java => DataSetCopier.java} | 8 +--
...nCopier.java => LocalAndRemoteCopier.java} | 61 ++++++++++++++++++-
...est.java => LocalAndRemoteCopierTest.java} | 28 +++++----
4 files changed, 83 insertions(+), 21 deletions(-)
rename rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/{MsInjectionDataSetCopier.java => DataSetCopier.java} (81%)
rename rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/{MsInjectionCopier.java => LocalAndRemoteCopier.java} (82%)
rename rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/{MsInjectionCopierTest.java => LocalAndRemoteCopierTest.java} (89%)
diff --git a/rtd_phosphonetx/etc/service.properties b/rtd_phosphonetx/etc/service.properties
index ae268dcd0a2..e77226919e3 100644
--- a/rtd_phosphonetx/etc/service.properties
+++ b/rtd_phosphonetx/etc/service.properties
@@ -131,8 +131,9 @@ copy-data-sets.class = ch.systemsx.cisd.openbis.dss.generic.server.plugins.stand
copy-data-sets.destination = vesuvio:tmp/${user}
ms-inj-copy.label = MS INJECTION Data Set Copier
-ms-inj-copy.dataset-types = -
-ms-inj-copy.class = ch.systemsx.cisd.openbis.dss.phosphonetx.server.plugins.MsInjectionDataSetCopier
+ms-inj-copy.dataset-types = UNKNOWN, RAW_DATA,MZXML_DATA,PROT_RESULT
+ms-inj-copy.class = ch.systemsx.cisd.openbis.dss.phosphonetx.server.plugins.DataSetCopier
+ms-inj-copy.marker-file-prefix = .MARKER_finished_
#ms-inj-copy.destination = /Volumes/share-1-\$/user/cisd/felmer/phosphonetx/
ms-inj-copy.destination = vesuvio:tmp
@@ -185,7 +186,7 @@ prot-result.incoming-data-completeness-condition = auto-detection
# ---------------- Plugin properties
# The extractor class to use for code extraction
-prot-result.data-set-info-extractor = ch.systemsx.cisd.openbis.etlserver.phosphonetx.DataSetInfoExtractorForSearchExperiment
+prot-result.data-set-info-extractor = ch.systemsx.cisd.openbis.etlserver.phosphonetx.DataSetInfoExtractorForProteinResults
prot-result.data-set-info-extractor.separator = +
# The extractor class to use for type extraction
prot-result.type-extractor = ch.systemsx.cisd.etlserver.SimpleTypeExtractor
diff --git a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionDataSetCopier.java b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/DataSetCopier.java
similarity index 81%
rename from rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionDataSetCopier.java
rename to rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/DataSetCopier.java
index b89716caebf..6b952e069da 100644
--- a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionDataSetCopier.java
+++ b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/DataSetCopier.java
@@ -31,19 +31,19 @@ import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.SshCommandEx
*
* @author Franz-Josef Elmer
*/
-public class MsInjectionDataSetCopier extends AbstractDropboxProcessingPlugin
+public class DataSetCopier extends AbstractDropboxProcessingPlugin
{
private static final long serialVersionUID = 1L;
- public MsInjectionDataSetCopier(Properties properties, File storeRoot)
+ public DataSetCopier(Properties properties, File storeRoot)
{
this(properties, storeRoot, new RsyncCopierFactory(), new SshCommandExecutorFactory());
}
- @Private MsInjectionDataSetCopier(Properties properties, File storeRoot, IPathCopierFactory pathCopierFactory,
+ @Private DataSetCopier(Properties properties, File storeRoot, IPathCopierFactory pathCopierFactory,
ISshCommandExecutorFactory sshCommandExecutorFactory)
{
- super(properties, storeRoot, new MsInjectionCopier(properties, pathCopierFactory,
+ super(properties, storeRoot, new LocalAndRemoteCopier(properties, pathCopierFactory,
sshCommandExecutorFactory));
}
}
diff --git a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionCopier.java b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopier.java
similarity index 82%
rename from rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionCopier.java
rename to rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopier.java
index 14e0627f6bb..2ed79d33074 100644
--- a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionCopier.java
+++ b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopier.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.openbis.dss.phosphonetx.server.plugins;
import java.io.File;
+import java.io.IOException;
import java.io.Serializable;
import java.util.List;
import java.util.Map;
@@ -25,6 +26,7 @@ import java.util.Properties;
import org.apache.log4j.Logger;
import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
import ch.systemsx.cisd.common.filesystem.FileOperations;
@@ -49,9 +51,11 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
* @author Franz-Josef Elmer
*/
-class MsInjectionCopier implements Serializable, IPostRegistrationDatasetHandler
+class LocalAndRemoteCopier implements Serializable, IPostRegistrationDatasetHandler
{
+ @Private static final String MARKER_FILE_PREFIX = "marker-file-prefix";
+
@Private
static final String SAMPLE_UNKNOWN = "sample-unknown";
@@ -87,6 +91,8 @@ class MsInjectionCopier implements Serializable, IPostRegistrationDatasetHandler
void copyDataSet(File dataSet, File destination);
void renameTo(File newFile, File oldFile);
+
+ void createMarkerFile(File markerFile);
}
private static final class LocalExcecutor implements IExecutor
@@ -144,6 +150,22 @@ class MsInjectionCopier implements Serializable, IPostRegistrationDatasetHandler
}
}
+ public void createMarkerFile(File markerFile)
+ {
+ try
+ {
+ boolean result = markerFile.createNewFile();
+ if (result == false)
+ {
+ throw new IOException("File '" + markerFile + "' already exists.");
+ }
+ } catch (IOException ex)
+ {
+ operationLog.error("Couldn't create marker file '" + markerFile + "'.", ex);
+ throw new ExceptionWithStatus(Status.createError("creating a marker file failed"), ex);
+ }
+ }
+
}
private static final class RemoteExecutor implements IExecutor
@@ -226,12 +248,34 @@ class MsInjectionCopier implements Serializable, IPostRegistrationDatasetHandler
throw new ExceptionWithStatus(Status.createError("moving leads to a problem"));
}
}
+
+ public void createMarkerFile(File markerFile)
+ {
+ ProcessResult result =
+ executor.executeCommandRemotely(
+ "touch " + markerFile.getPath(),
+ DataSetCopier.SSH_TIMEOUT_MILLIS);
+ if (result.isOK() == false)
+ {
+ operationLog.error("Creation of marker file '" + markerFile
+ + "' failed with exit value: " + result.getExitValue());
+ throw new ExceptionWithStatus(Status.createError("creating a marker file failed"));
+ }
+ List<String> output = result.getOutput();
+ if (output.isEmpty() == false)
+ {
+ operationLog.error("Creation of marker file '" + markerFile
+ + "' seemed to be successful but produced following output:\n"
+ + StringUtilities.concatenateWithNewLine(output));
+ throw new ExceptionWithStatus(Status.createError("creating a marker file leads to a problem"));
+ }
+ }
}
private static final long serialVersionUID = 1L;
private final static Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
- MsInjectionCopier.class);
+ LocalAndRemoteCopier.class);
private final Properties properties;
@@ -243,12 +287,19 @@ class MsInjectionCopier implements Serializable, IPostRegistrationDatasetHandler
private transient File destination;
- MsInjectionCopier(Properties properties, IPathCopierFactory pathCopierFactory,
+ private String markerFilePrefix;
+
+ LocalAndRemoteCopier(Properties properties, IPathCopierFactory pathCopierFactory,
ISshCommandExecutorFactory sshCommandExecutorFactory)
{
this.properties = properties;
this.pathCopierFactory = pathCopierFactory;
this.sshCommandExecutorFactory = sshCommandExecutorFactory;
+ markerFilePrefix = properties.getProperty(MARKER_FILE_PREFIX);
+ if (markerFilePrefix != null && markerFilePrefix.length() == 0)
+ {
+ throw new ConfigurationFailureException("marker-file-prefix is an empty string.");
+ }
init();
}
@@ -303,6 +354,10 @@ class MsInjectionCopier implements Serializable, IPostRegistrationDatasetHandler
deleteTargetFolder(targetFolder);
executor.copyDataSet(originalData, destination);
executor.renameTo(targetFolder, new File(destination, originalData.getName()));
+ if (markerFilePrefix != null)
+ {
+ executor.createMarkerFile(new File(destination, markerFilePrefix + target));
+ }
return Status.OK;
} catch (ExceptionWithStatus ex)
diff --git a/rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionCopierTest.java b/rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopierTest.java
similarity index 89%
rename from rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionCopierTest.java
rename to rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopierTest.java
index 02e084c501c..da4303bf4e5 100644
--- a/rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/MsInjectionCopierTest.java
+++ b/rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/dss/phosphonetx/server/plugins/LocalAndRemoteCopierTest.java
@@ -46,8 +46,8 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
/**
* @author Franz-Josef Elmer
*/
-@Friend(toClasses = MsInjectionCopier.class)
-public class MsInjectionCopierTest extends AbstractFileSystemTestCase
+@Friend(toClasses = LocalAndRemoteCopier.class)
+public class LocalAndRemoteCopierTest extends AbstractFileSystemTestCase
{
private static final ProcessResult OK_RESULT = new ProcessResult(Arrays.asList(""), 0, null,
null, 0, null, null, null);
@@ -115,8 +115,8 @@ public class MsInjectionCopierTest extends AbstractFileSystemTestCase
{
Properties properties = new Properties();
properties.setProperty(DataSetCopier.DESTINATION_KEY, destination.getPath());
- MsInjectionCopier msInjectionCopier =
- new MsInjectionCopier(properties, copierFactory, sshExecutorFactory);
+ LocalAndRemoteCopier msInjectionCopier =
+ new LocalAndRemoteCopier(properties, copierFactory, sshExecutorFactory);
prepareForCheckingLastModifiedDate();
DataSetInformation dataSetInformation = new DataSetInformation();
@@ -141,8 +141,8 @@ public class MsInjectionCopierTest extends AbstractFileSystemTestCase
{
Properties properties = new Properties();
properties.setProperty(DataSetCopier.DESTINATION_KEY, destination.getPath());
- MsInjectionCopier msInjectionCopier =
- new MsInjectionCopier(properties, copierFactory, sshExecutorFactory);
+ LocalAndRemoteCopier msInjectionCopier =
+ new LocalAndRemoteCopier(properties, copierFactory, sshExecutorFactory);
DataSetInformation dataSetInformation = new DataSetInformation();
dataSetInformation.setDataSetCode(DATA_SET_CODE);
@@ -162,8 +162,8 @@ public class MsInjectionCopierTest extends AbstractFileSystemTestCase
{
Properties properties = new Properties();
properties.setProperty(DataSetCopier.DESTINATION_KEY, destination.getPath());
- MsInjectionCopier msInjectionCopier =
- new MsInjectionCopier(properties, copierFactory, sshExecutorFactory);
+ LocalAndRemoteCopier msInjectionCopier =
+ new LocalAndRemoteCopier(properties, copierFactory, sshExecutorFactory);
File copiedDataSet = new File(destination, FOLDER_NAME);
copiedDataSet.mkdirs();
File dummy = new File(copiedDataSet, "dummy");
@@ -186,9 +186,10 @@ public class MsInjectionCopierTest extends AbstractFileSystemTestCase
}
@Test
- public void testRemote()
+ public void testRemoteWithMarkerFile()
{
Properties properties = new Properties();
+ properties.setProperty(LocalAndRemoteCopier.MARKER_FILE_PREFIX, "MARKER-");
properties.setProperty(DataSetCopier.DESTINATION_KEY, "localhost:" + destination.getPath());
properties.setProperty(DataSetCopier.RSYNC_EXEC + "-executable", rsyncExec.getPath());
properties.setProperty(DataSetCopier.SSH_EXEC + "-executable", sshExec.getPath());
@@ -221,10 +222,15 @@ public class MsInjectionCopierTest extends AbstractFileSystemTestCase
"mv " + new File(destination, dataSet.getName()) + " " + copiedDataSet,
SSH_TIMEOUT_MILLIS);
will(returnValue(OK_RESULT));
+
+ one(sshExecutor).executeCommandRemotely(
+ "touch " + new File(destination, "MARKER-" + FOLDER_NAME),
+ SSH_TIMEOUT_MILLIS);
+ will(returnValue(OK_RESULT));
}
});
- MsInjectionCopier msInjectionCopier =
- new MsInjectionCopier(properties, copierFactory, sshExecutorFactory);
+ LocalAndRemoteCopier msInjectionCopier =
+ new LocalAndRemoteCopier(properties, copierFactory, sshExecutorFactory);
DataSetInformation dataSetInformation = new DataSetInformation();
dataSetInformation.setDataSetCode(DATA_SET_CODE);
--
GitLab