From edebe33609eeadca09c97997dbc8f6b587f48a91 Mon Sep 17 00:00:00 2001
From: kohleman <kohleman>
Date: Wed, 7 Sep 2016 15:16:43 +0000
Subject: [PATCH] - added rsync parameter fields - allow tracking of samples
without sending an email (new flag in command line)
SVN: 37038
---
.../cisd/dsu/tracking/main/Parameters.java | 20 ++++++
.../cisd/dsu/tracking/main/TrackingBO.java | 71 +++++++++----------
.../dsu/tracking/main/TrackingClient.java | 25 +++++++
.../cisd/dsu/tracking/utils/LogUtils.java | 2 +-
4 files changed, 81 insertions(+), 37 deletions(-)
diff --git a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/Parameters.java b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/Parameters.java
index 188e8d55d98..562fd244815 100644
--- a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/Parameters.java
+++ b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/Parameters.java
@@ -55,6 +55,10 @@ public class Parameters
private static final String DESTINATION_FOLDER = "destination-folder";
+ private static final String RSYNC_BINARY = "rsync-binary";
+
+ private static final String RSYNC_FLAGS = "rsync-flags";
+
private final String openbisUser;
private final String openbisPassword;
@@ -80,6 +84,10 @@ public class Parameters
private final String dataSetTypeList;
private final String destinationFolder;
+
+ private final String rsyncBinary;
+
+ private final String rsyncFlags;
public Parameters(Properties props)
{
@@ -96,6 +104,8 @@ public class Parameters
this.oldDataSetBacklogNumber = PropertyUtils.getInt(props, OLD_DATA_SET_BACKLOG_NUMBER, 0);
this.dataSetTypeList = PropertyUtils.getProperty(props, DATA_SET_TYPE_LIST);
this.destinationFolder= PropertyUtils.getProperty(props, DESTINATION_FOLDER);
+ this.rsyncBinary = PropertyUtils.getProperty(props, RSYNC_BINARY);
+ this.rsyncFlags = PropertyUtils.getProperty(props, RSYNC_FLAGS);
}
public String getOpenbisUser()
@@ -162,4 +172,14 @@ public class Parameters
{
return destinationFolder;
}
+
+ public String getRsyncBinary()
+ {
+ return rsyncBinary;
+ }
+
+ public String getRsyncFlags()
+ {
+ return rsyncFlags;
+ }
}
\ No newline at end of file
diff --git a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/TrackingBO.java b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/TrackingBO.java
index aae72a17996..a958dfcd367 100644
--- a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/TrackingBO.java
+++ b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/TrackingBO.java
@@ -96,9 +96,19 @@ public class TrackingBO
if (commandLineMap.get(TrackingClient.CL_PARAMETER_LANES) != null)
{
+ String[] laneCodeList = commandLineMap.get(TrackingClient.CL_PARAMETER_LANES);
changedEntities = fetchChangedDataSets(prevTrackingState, trackingServer, params,
- commandLineMap.get(TrackingClient.CL_PARAMETER_LANES), session);
+ commandLineMap, laneCodeList, session);
+ }
+
+ else if (commandLineMap.get(TrackingClient.CL_PARAMETER_REMOVE_LANES) != null)
+ {
+ sendEmails = false;
+ String[] laneCodeList = commandLineMap.get(TrackingClient.CL_PARAMETER_REMOVE_LANES);
+ changedEntities = fetchChangedDataSets(prevTrackingState, trackingServer, params,
+ commandLineMap, laneCodeList, session);
}
+
else if (commandLineMap.containsKey(TrackingClient.CL_PARAMETER_ALL))
{
@@ -106,7 +116,7 @@ public class TrackingBO
System.out.println("This function is deactivated");
}
- // just list the potential chnaged lanes
+ // just list the potential changed lanes
else if (commandLineMap.containsKey(TrackingClient.CL_PARAMETER_CHANGED_LANES))
{
Map<String, String> changed_lanes = fetchChangedLanes(prevTrackingState, trackingServer, params, session);
@@ -282,24 +292,6 @@ public class TrackingBO
}
}
-
-
-// Properties properties = new Properties();
-// File storeRoot = null;
-//
-// DataSetCopier dsc = new DataSetCopier(properties, storeRoot);
-// dsc.process(description, context);
-
-// PhysicalDataSet pds = d.tryGetAsDataSet();
-//
-// DataSetProcessingContext context = null;
-//
-// DatasetDescription description = new DatasetDescription();
-// description.setDataSetCode(d.getCode());
-// description.setDatasetTypeCode(d.getDataSetType().getCode());
-// description.setDataSetLocation(d.tryGetAsDataSet().getDataSetLocation());
-
-
Set<Map.Entry<String, String>> entrySet = changedLanesMap.entrySet();
for (Entry<String, String> entry : entrySet)
{
@@ -311,9 +303,11 @@ public class TrackingBO
}
private static TrackedEntities fetchChangedDataSets(TrackingStateDTO trackingState,
- ITrackingServer trackingServer, Parameters params, String[] laneCodeList, SessionContextDTO session)
+ ITrackingServer trackingServer, Parameters params, final HashMap<String, String[]> commandLineMap,
+ String[] laneCodeList, SessionContextDTO session)
{
long usableDataSetId = getUsableDataSetId(trackingState, params);
+
List<String> spaceWhiteList = Arrays.asList(params.getSpaceWhitelist().split("\\s*,\\s*"));
List<String> datasetTypeList = Arrays.asList(params.getdataSetTypeList().split("\\s*,\\s*"));
@@ -361,29 +355,34 @@ public class TrackingBO
}
}
}
- System.out.println("Sending Email for " + newDataSetID + " which is part of " + d.getSampleCode());
+// System.out.println("Sending Email for Data Set with permID " + d.getPermId() + " which is part of " + d.getSampleCode());
addDataSetTo(changedTrackingMap, d);
}
- LogUtils.info("TO_TRANSFER: Found " + toTransferDataSets.size() + " data sets which are to be transferred to an extra folder");
-
-
-
- //TODO: extract and make robust
- File rsyncBinary = new File("/opt/local/bin/rsync");
- File destination = new File(params.getDestinationFolder());
- RsyncCopier copier = new RsyncCopier(rsyncBinary, null, "-a");
+ LogUtils.info("TO_TRANSFER: Found " + toTransferDataSets.size() + " data sets which could be transferred to an extra folder");
+
- for (AbstractExternalData ds : dataSets) {
- File source = new File(ds.tryGetAsDataSet().getFullLocation());
- copier.copy(source, destination, null, null);
- LogUtils.info("Copying " + ds.getCode() + " to " + params.getDestinationFolder());
-
+ if (commandLineMap.containsKey(TrackingClient.CL_PARAMETER_COPY_DATA_SETS))
+ {
+ extraDataSetCopy(params, toTransferDataSets);
}
LogUtils.info("Found " + filteredDataSets.size() + " data sets which are connected to samples in " + filterList.toString());
return gatherTrackedEntities(trackingState, trackingServer, session, filteredDataSets, changedTrackingMap);
}
+
+
+ private static void extraDataSetCopy(Parameters params, List<AbstractExternalData> dataSets) {
+ File rsyncBinary = new File(params.getRsyncBinary());
+ File destination = new File(params.getDestinationFolder());
+ RsyncCopier copier = new RsyncCopier(rsyncBinary, null, params.getRsyncFlags());
+
+ for (AbstractExternalData ds : dataSets) {
+ File source = new File(ds.tryGetAsDataSet().getFullLocation());
+ copier.copy(source, destination, null, null);
+ LogUtils.info("Copying " + ds.getCode() + " from " + source + " to " + params.getDestinationFolder());
+ }
+ }
private static long getMaxDataSetId(TrackingStateDTO trackingState)
{
@@ -440,7 +439,7 @@ public class TrackingBO
Sample currentLane = dataSet.getSample();
String lanePermId = currentLane.getPermId();
- LogUtils.info("Found lane with permId: " + lanePermId + " with new DS techId " + dataSet.getId() + " and DS permId " + dataSet.getPermId());
+ LogUtils.info("Found lane " + currentLane.getCode() + " with permId: " + lanePermId + " with new DS techId " + dataSet.getId() + " and DS permId " + dataSet.getPermId());
ArrayList<Long> existingList = changedTrackingMap.get(lanePermId);
if (existingList == null)
{
diff --git a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/TrackingClient.java b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/TrackingClient.java
index c7a246076ef..617b7a897cc 100644
--- a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/TrackingClient.java
+++ b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/main/TrackingClient.java
@@ -71,6 +71,10 @@ public class TrackingClient
public static final String CL_PARAMETER_CHANGED_LANES = "changed_lanes";
+ public static final String CL_PARAMETER_COPY_DATA_SETS = "copy_data_sets";
+
+ public static final String CL_PARAMETER_REMOVE_LANES = "remove";
+
public static void main(String[] args)
{
try
@@ -98,11 +102,22 @@ public class TrackingClient
.create(CL_PARAMETER_LANES);
lanes.setArgs(Option.UNLIMITED_VALUES);
// Option all = new Option(CL_PARAMETER_ALL, "track all lanes, only for testing, never use in production!");
+
Option new_lanes = new Option(CL_PARAMETER_CHANGED_LANES, "only list lanes which have new datasets");
+
+ Option copy_data_sets = new Option(CL_PARAMETER_COPY_DATA_SETS, "also copy the corresponding data sets to an extra"
+ + " folder. Only in combination with parameter \"" + CL_PARAMETER_LANES + "\"");
+
+ Option remove = OptionBuilder.withArgName(CL_PARAMETER_REMOVE_LANES)
+ .hasArg()
+ .withDescription("remove lanes from tracking list and do not send an email")
+ .create(CL_PARAMETER_REMOVE_LANES);
options.addOption(lanes);
// options.addOption(all);
options.addOption(new_lanes);
+ options.addOption(copy_data_sets);
+ options.addOption(remove);
// automatically generate the help statement
HelpFormatter formatter = new HelpFormatter();
@@ -130,9 +145,19 @@ public class TrackingClient
{
commandLineMap.put(CL_PARAMETER_CHANGED_LANES, null);
}
+ if (line.hasOption(CL_PARAMETER_COPY_DATA_SETS))
+ {
+ commandLineMap.put(CL_PARAMETER_COPY_DATA_SETS, null);
+ }
+ if (line.hasOption(CL_PARAMETER_REMOVE_LANES))
+ {
+ commandLineMap.put(CL_PARAMETER_REMOVE_LANES, line.getOptionValues(CL_PARAMETER_REMOVE_LANES));
+ }
} catch (ParseException exp)
{
LogUtils.environmentError("Parsing of command line parameters failed.", exp.getMessage());
+ System.out.println("Parsing of command line parameters failed. " + exp.getMessage());
+ System.exit(1);
}
return commandLineMap;
}
diff --git a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/utils/LogUtils.java b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/utils/LogUtils.java
index 1e123289452..1121a7d570c 100644
--- a/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/utils/LogUtils.java
+++ b/deep_sequencing_unit/source/java/ch/ethz/bsse/cisd/dsu/tracking/utils/LogUtils.java
@@ -47,7 +47,7 @@ public class LogUtils
public static void notify(Throwable ex)
{
String fullMsg =
- "An unexpected exception occured why trying to send emails with changes.\n"
+ "An unexpected exception occured while trying to send emails with changes.\n"
+ "Error details: " + ex.getMessage();
notify(ex, fullMsg);
}
--
GitLab