From 823b51321f6c7cfd3d3448a15a0e44458cf6df5f Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Thu, 30 May 2013 05:49:10 +0000
Subject: [PATCH] SP-668, BIS-428: Refactoring UploadingCommand by introducing
AbstractDataSetPackager and ZipDataSetPackager. Extracting MetaDataBuilder as
outerclass.
SVN: 29248
---
.../server/AbstractDataSetPackager.java | 121 ++++++++
.../dss/generic/server/MetaDataBuilder.java | 241 +++++++++++++++
.../dss/generic/server/UploadingCommand.java | 288 +-----------------
.../generic/server/ZipDataSetPackager.java | 81 +++++
.../AbstractArchiverProcessingPlugin.java | 2 +-
.../plugins/standard/DistributedArchiver.java | 86 ++++++
6 files changed, 537 insertions(+), 282 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AbstractDataSetPackager.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/MetaDataBuilder.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ZipDataSetPackager.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedArchiver.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AbstractDataSetPackager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AbstractDataSetPackager.java
new file mode 100644
index 00000000000..db88e82edce
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AbstractDataSetPackager.java
@@ -0,0 +1,121 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server;
+
+import java.io.ByteArrayInputStream;
+import java.io.InputStream;
+import java.util.List;
+
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.logging.LogLevel;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.translator.DataSetTranslator;
+
+/**
+ * Abstract super class of packagers which package all files of a data set together with a meta data file.
+ *
+ * @author Franz-Josef Elmer
+ */
+public abstract class AbstractDataSetPackager
+{
+ public static final String META_DATA_FILE_NAME = "meta-data.tsv";
+
+ private final ISimpleLogger logger;
+ private final IHierarchicalContentProvider contentProvider;
+ private final DataSetExistenceChecker dataSetExistenceChecker;
+
+ protected AbstractDataSetPackager(ISimpleLogger logger, IHierarchicalContentProvider contentProvider,
+ DataSetExistenceChecker dataSetExistenceChecker)
+ {
+ this.logger = logger;
+ this.contentProvider = contentProvider;
+ this.dataSetExistenceChecker = dataSetExistenceChecker;
+ }
+
+ public boolean addDataSetTo(String rootPath, AbstractExternalData externalData)
+ {
+ try
+ {
+ addEntry(rootPath + META_DATA_FILE_NAME,
+ System.currentTimeMillis(),
+ new ByteArrayInputStream(MetaDataBuilder.createMetaData(externalData).getBytes()));
+ } catch (Exception ex)
+ {
+ logger.log(LogLevel.ERROR,
+ "Couldn't add meta data for data set '" + externalData.getCode()
+ + "' to zip file.", ex);
+ return false;
+ }
+ if (dataSetExistenceChecker.dataSetExists(DataSetTranslator
+ .translateToDescription(externalData)) == false)
+ {
+ return handleNonExistingDataSet(externalData, null);
+ }
+ IHierarchicalContent root = null;
+ try
+ {
+ root = contentProvider.asContent(externalData.getCode());
+ } catch (Exception ex)
+ {
+ return handleNonExistingDataSet(externalData, ex);
+ }
+ try
+ {
+ addTo(rootPath, root.getRootNode());
+ return true;
+ } catch (Exception ex)
+ {
+ logger.log(LogLevel.ERROR, "Couldn't add data set '" + externalData.getCode()
+ + "' to zip file.", ex);
+ return false;
+ } finally
+ {
+ if (root != null)
+ {
+ root.close();
+ }
+ }
+ }
+
+ private boolean handleNonExistingDataSet(AbstractExternalData externalData, Exception ex)
+ {
+ logger.log(LogLevel.ERROR, "Data set " + externalData.getCode() + " does not exist.", ex);
+ return false;
+ }
+
+ private void addTo(String newRootPath, IHierarchicalContentNode node)
+ {
+ if (node.isDirectory())
+ {
+ List<IHierarchicalContentNode> childNodes = node.getChildNodes();
+ for (IHierarchicalContentNode childNode : childNodes)
+ {
+ addTo(newRootPath, childNode);
+ }
+ } else
+ {
+ addEntry(newRootPath + node.getRelativePath(), node.getLastModified(),
+ node.getInputStream());
+ }
+ }
+
+ public abstract void addEntry(String entryPath, long lastModified, InputStream in);
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/MetaDataBuilder.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/MetaDataBuilder.java
new file mode 100644
index 00000000000..adb91cd1ccb
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/MetaDataBuilder.java
@@ -0,0 +1,241 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server;
+
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.Date;
+import java.util.List;
+
+import ch.systemsx.cisd.openbis.common.types.BooleanOrUnknown;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Person;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space;
+
+/**
+ * Helper class to render a data set with properties as multi-line text.
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class MetaDataBuilder
+{
+ private static final String DATA_SET = "data_set";
+
+ private static final String SAMPLE = "sample";
+
+ private static final String EXPERIMENT = "experiment";
+
+ private static final char DELIM = '\t';
+
+ private static final DateFormat DATE_FORMAT_PATTERN = new SimpleDateFormat(
+ "yyyy-MM-dd HH:mm:ss Z");
+
+ private static final Comparator<AbstractExternalData> DATA_SET_COMPARATOR = new Comparator<AbstractExternalData>()
+ {
+ @Override
+ public int compare(AbstractExternalData d1, AbstractExternalData d2)
+ {
+ return d1.getCode().compareTo(d2.getCode());
+ }
+ };
+
+ private static final Comparator<IEntityProperty> PROPERTIES_COMPARATOR = new Comparator<IEntityProperty>()
+ {
+ @Override
+ public int compare(IEntityProperty p1, IEntityProperty p2)
+ {
+ return p1.getPropertyType().getCode().compareTo(p2.getPropertyType().getCode());
+ }
+ };
+
+ public static String createMetaData(AbstractExternalData dataSet)
+ {
+ MetaDataBuilder builder = new MetaDataBuilder();
+ builder.dataSet("code", dataSet.getCode());
+ builder.dataSet("production_timestamp", dataSet.getProductionDate());
+ builder.dataSet("producer_code", dataSet.getDataProducerCode());
+ builder.dataSet("data_set_type", dataSet.getDataSetType().getCode());
+ builder.dataSet("is_measured", dataSet.isDerived() == false);
+ if (dataSet.tryGetAsDataSet() != null)
+ {
+ final Boolean completeFlag = dataSet.tryGetAsDataSet().getComplete();
+ builder.dataSet("is_complete", BooleanOrUnknown.T.equals(completeFlag));
+ }
+ builder.dataSetProperties(dataSet.getProperties());
+
+ StringBuilder stringBuilder = new StringBuilder();
+ List<AbstractExternalData> parents = new ArrayList<AbstractExternalData>(dataSet.getParents());
+ Collections.sort(parents, DATA_SET_COMPARATOR);
+ if (parents.isEmpty() == false)
+ {
+ for (AbstractExternalData parent : parents)
+ {
+ if (stringBuilder.length() > 0)
+ {
+ stringBuilder.append(',');
+ }
+ stringBuilder.append(parent.getCode());
+ }
+ }
+ builder.dataSet("parent_codes", stringBuilder.toString());
+ Sample sample = dataSet.getSample();
+ if (sample != null)
+ {
+ builder.sample("type_code", sample.getSampleType().getCode());
+ builder.sample("code", sample.getCode());
+ Space space = sample.getSpace();
+ builder.sample("space_code", space == null ? "(shared)" : space.getCode());
+ builder.sample("registration_timestamp", sample.getRegistrationDate());
+ builder.sample("registrator", sample.getRegistrator());
+ builder.sampleProperties(sample.getProperties());
+ }
+ Experiment experiment = dataSet.getExperiment();
+ Project project = experiment.getProject();
+ builder.experiment("space_code", project.getSpace().getCode());
+ builder.experiment("project_code", project.getCode());
+ builder.experiment("experiment_code", experiment.getCode());
+ builder.experiment("experiment_type_code", experiment.getExperimentType().getCode());
+ builder.experiment("registration_timestamp", experiment.getRegistrationDate());
+ builder.experiment("registrator", experiment.getRegistrator());
+ builder.experimentProperties(experiment.getProperties());
+ return builder.getRenderedMetaData();
+ }
+
+ private final StringBuilder builder = new StringBuilder();
+
+ private void dataSetProperties(List<IEntityProperty> properties)
+ {
+ addProperties(DATA_SET, properties);
+ }
+
+ private void sampleProperties(List<IEntityProperty> properties)
+ {
+ addProperties(SAMPLE, properties);
+ }
+
+ private void experimentProperties(List<IEntityProperty> properties)
+ {
+ addProperties(EXPERIMENT, properties);
+ }
+
+ private void addProperties(String category, List<IEntityProperty> properties)
+ {
+ Collections.sort(properties, PROPERTIES_COMPARATOR);
+ for (IEntityProperty property : properties)
+ {
+ addRow(category, property.getPropertyType().getCode(), property.tryGetAsString());
+ }
+ }
+
+ private void dataSet(String key, String value)
+ {
+ addRow(DATA_SET, key, value);
+ }
+
+ private void dataSet(String key, Date date)
+ {
+ addRow(DATA_SET, key, date);
+ }
+
+ private void dataSet(String key, boolean flag)
+ {
+ addRow(DATA_SET, key, flag);
+ }
+
+ private void sample(String key, String value)
+ {
+ addRow(SAMPLE, key, value);
+ }
+
+ private void sample(String key, Person person)
+ {
+ addRow(SAMPLE, key, person);
+ }
+
+ private void sample(String key, Date date)
+ {
+ addRow(SAMPLE, key, date);
+ }
+
+ private void experiment(String key, String value)
+ {
+ addRow(EXPERIMENT, key, value);
+ }
+
+ private void experiment(String key, Person person)
+ {
+ addRow(EXPERIMENT, key, person);
+ }
+
+ private void experiment(String key, Date date)
+ {
+ addRow(EXPERIMENT, key, date);
+ }
+
+ private void addRow(String category, String key, Person person)
+ {
+ StringBuilder stringBuilder = new StringBuilder();
+ if (person != null)
+ {
+ String firstName = person.getFirstName();
+ String lastName = person.getLastName();
+ if (firstName != null && lastName != null)
+ {
+ stringBuilder.append(firstName).append(' ').append(lastName);
+ } else
+ {
+ stringBuilder.append(person.getUserId());
+ }
+ String email = person.getEmail();
+ if (email != null)
+ {
+ stringBuilder.append(" <").append(email).append(">");
+ }
+ }
+ addRow(category, key, stringBuilder.toString());
+ }
+
+ private void addRow(String category, String key, Date date)
+ {
+ addRow(category, key, date == null ? null : DATE_FORMAT_PATTERN.format(date));
+ }
+
+ private void addRow(String category, String key, boolean flag)
+ {
+ addRow(category, key, Boolean.valueOf(flag).toString().toUpperCase());
+ }
+
+ private void addRow(String category, String key, String value)
+ {
+ builder.append(category).append(DELIM).append(key).append(DELIM);
+ builder.append(value == null ? "" : value).append('\n');
+ }
+
+ private String getRenderedMetaData()
+ {
+ return builder.toString();
+ }
+
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java
index eb6120454c1..a4a2d0106a6 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java
@@ -16,22 +16,15 @@
package ch.systemsx.cisd.openbis.dss.generic.server;
-import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
-import java.io.InputStream;
import java.io.OutputStream;
-import java.text.DateFormat;
-import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
-import java.util.Collection;
-import java.util.Date;
import java.util.List;
import java.util.Properties;
-import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
@@ -41,6 +34,7 @@ import ch.systemsx.cisd.cifex.rpc.client.ICIFEXComponent;
import ch.systemsx.cisd.cifex.rpc.client.ICIFEXUploader;
import ch.systemsx.cisd.cifex.rpc.client.gui.IProgressListener;
import ch.systemsx.cisd.cifex.shared.basic.Constants;
+import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.mail.IMailClient;
@@ -48,24 +42,16 @@ import ch.systemsx.cisd.common.mail.MailClient;
import ch.systemsx.cisd.common.mail.MailClientParameters;
import ch.systemsx.cisd.common.security.TokenGenerator;
import ch.systemsx.cisd.common.time.TimingParameters;
-import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
-import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
-import ch.systemsx.cisd.openbis.common.types.BooleanOrUnknown;
import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Person;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetUploadContext;
-import ch.systemsx.cisd.openbis.generic.shared.translator.DataSetTranslator;
-import de.schlichtherle.util.zip.ZipEntry;
import de.schlichtherle.util.zip.ZipOutputStream;
/**
@@ -139,135 +125,6 @@ class UploadingCommand implements IDataSetCommand
}
- private static final class MetaDataBuilder
- {
- private static final String DATA_SET = "data_set";
-
- private static final String SAMPLE = "sample";
-
- private static final String EXPERIMENT = "experiment";
-
- private static final char DELIM = '\t';
-
- private static final DateFormat DATE_FORMAT_PATTERN = new SimpleDateFormat(
- "yyyy-MM-dd HH:mm:ss Z");
-
- private final StringBuilder builder = new StringBuilder();
-
- void dataSetProperties(List<IEntityProperty> properties)
- {
- addProperties(DATA_SET, properties);
- }
-
- void sampleProperties(List<IEntityProperty> properties)
- {
- addProperties(SAMPLE, properties);
- }
-
- void experimentProperties(List<IEntityProperty> properties)
- {
- addProperties(EXPERIMENT, properties);
- }
-
- void addProperties(String category, List<IEntityProperty> properties)
- {
- for (IEntityProperty property : properties)
- {
- addRow(category, property.getPropertyType().getCode(), property.tryGetAsString());
- }
- }
-
- void dataSet(String key, String value)
- {
- addRow(DATA_SET, key, value);
- }
-
- void dataSet(String key, Date date)
- {
- addRow(DATA_SET, key, date);
- }
-
- void dataSet(String key, boolean flag)
- {
- addRow(DATA_SET, key, flag);
- }
-
- void sample(String key, String value)
- {
- addRow(SAMPLE, key, value);
- }
-
- void sample(String key, Person person)
- {
- addRow(SAMPLE, key, person);
- }
-
- void sample(String key, Date date)
- {
- addRow(SAMPLE, key, date);
- }
-
- void experiment(String key, String value)
- {
- addRow(EXPERIMENT, key, value);
- }
-
- void experiment(String key, Person person)
- {
- addRow(EXPERIMENT, key, person);
- }
-
- void experiment(String key, Date date)
- {
- addRow(EXPERIMENT, key, date);
- }
-
- private void addRow(String category, String key, Person person)
- {
- StringBuilder stringBuilder = new StringBuilder();
- if (person != null)
- {
- String firstName = person.getFirstName();
- String lastName = person.getLastName();
- if (firstName != null && lastName != null)
- {
- stringBuilder.append(firstName).append(' ').append(lastName);
- } else
- {
- stringBuilder.append(person.getUserId());
- }
- String email = person.getEmail();
- if (email != null)
- {
- stringBuilder.append(" <").append(email).append(">");
- }
- }
- addRow(category, key, stringBuilder.toString());
- }
-
- private void addRow(String category, String key, Date date)
- {
- addRow(category, key, date == null ? null : DATE_FORMAT_PATTERN.format(date));
- }
-
- private void addRow(String category, String key, boolean flag)
- {
- addRow(category, key, Boolean.valueOf(flag).toString().toUpperCase());
- }
-
- private void addRow(String category, String key, String value)
- {
- builder.append(category).append(DELIM).append(key).append(DELIM);
- builder.append(value == null ? "" : value).append('\n');
- }
-
- @Override
- public String toString()
- {
- return builder.toString();
- }
- }
-
private final ICIFEXRPCServiceFactory cifexServiceFactory;
private final List<AbstractExternalData> dataSets;
@@ -430,52 +287,20 @@ class UploadingCommand implements IDataSetCommand
DataSetExistenceChecker dataSetExistenceChecker =
new DataSetExistenceChecker(dataSetDirectoryProvider,
TimingParameters.create(new Properties()));
+ Log4jSimpleLogger logger = new Log4jSimpleLogger(notificationLog);
+ ZipDataSetPackager packager = new ZipDataSetPackager(zipOutputStream, true,
+ logger, getHierarchicalContentProvider(), dataSetExistenceChecker);
for (AbstractExternalData externalData : dataSets)
{
String newRootPath = createRootPath(externalData) + "/";
- try
- {
- addEntry(zipOutputStream, newRootPath + "meta-data.tsv",
- System.currentTimeMillis(),
- new ByteArrayInputStream(createMetaData(externalData).getBytes()));
- } catch (IOException ex)
- {
- notificationLog.error(
- "Couldn't add meta date for data set '" + externalData.getCode()
- + "' to zip file.", ex);
- return false;
- }
- if (dataSetExistenceChecker.dataSetExists(DataSetTranslator
- .translateToDescription(externalData)) == false)
- {
- return handleNonExistingDataSet(externalData, null);
- }
- IHierarchicalContent root = null;
- try
- {
- root = getHierarchicalContentProvider().asContent(externalData.getCode());
- } catch (Exception ex)
- {
- return handleNonExistingDataSet(externalData, ex);
- }
- try
- {
- addTo(zipOutputStream, newRootPath, root.getRootNode());
- } catch (IOException ex)
+ boolean success = packager.addDataSetTo(newRootPath, externalData);
+ if (success == false)
{
- notificationLog.error("Couldn't add data set '" + externalData.getCode()
- + "' to zip file.", ex);
return false;
- } finally
- {
- if (root != null)
- {
- root.close();
- }
}
}
return true;
- } catch (IOException ex)
+ } catch (Exception ex)
{
notificationLog.error("Couldn't create zip file for uploading", ex);
return false;
@@ -494,13 +319,6 @@ class UploadingCommand implements IDataSetCommand
}
}
- private boolean handleNonExistingDataSet(AbstractExternalData externalData, Exception ex)
- {
- notificationLog.error(
- "Data set " + externalData.getCode() + " does not exist.", ex);
- return false;
- }
-
private IHierarchicalContentProvider getHierarchicalContentProvider()
{
if (hierarchicalContentProvider == null)
@@ -510,76 +328,6 @@ class UploadingCommand implements IDataSetCommand
return hierarchicalContentProvider;
}
- private void addTo(ZipOutputStream zipOutputStream, String newRootPath,
- IHierarchicalContentNode node) throws IOException
- {
- if (node.isDirectory())
- {
- List<IHierarchicalContentNode> childNodes = node.getChildNodes();
- for (IHierarchicalContentNode childNode : childNodes)
- {
- addTo(zipOutputStream, newRootPath, childNode);
- }
- } else
- {
- addEntry(zipOutputStream, newRootPath + node.getRelativePath(), node.getLastModified(),
- node.getInputStream());
- }
- }
-
- private String createMetaData(AbstractExternalData dataSet)
- {
- MetaDataBuilder builder = new MetaDataBuilder();
- builder.dataSet("code", dataSet.getCode());
- builder.dataSet("production_timestamp", dataSet.getProductionDate());
- builder.dataSet("producer_code", dataSet.getDataProducerCode());
- builder.dataSet("data_set_type", dataSet.getDataSetType().getCode());
- builder.dataSet("is_measured", dataSet.isDerived() == false);
- if (dataSet.tryGetAsDataSet() != null)
- {
- final Boolean completeFlag = dataSet.tryGetAsDataSet().getComplete();
- builder.dataSet("is_complete", BooleanOrUnknown.T.equals(completeFlag));
- }
- builder.dataSetProperties(dataSet.getProperties());
-
- StringBuilder stringBuilder = new StringBuilder();
- Collection<AbstractExternalData> parents = dataSet.getParents();
- if (parents.isEmpty() == false)
- {
- for (AbstractExternalData parent : parents)
- {
- if (stringBuilder.length() > 0)
- {
- stringBuilder.append(',');
- }
- stringBuilder.append(parent.getCode());
- }
- }
- builder.dataSet("parent_codes", stringBuilder.toString());
- Sample sample = dataSet.getSample();
- if (sample != null)
- {
- builder.sample("type_code", sample.getSampleType().getCode());
- builder.sample("code", sample.getCode());
- Space space = sample.getSpace();
- builder.sample("space_code", space == null ? "(shared)" : space.getCode());
- // group->space
- builder.sample("registration_timestamp", sample.getRegistrationDate());
- builder.sample("registrator", sample.getRegistrator());
- builder.sampleProperties(sample.getProperties());
- }
- Experiment experiment = dataSet.getExperiment();
- Project project = experiment.getProject();
- builder.experiment("space_code", project.getSpace().getCode());
- builder.experiment("project_code", project.getCode());
- builder.experiment("experiment_code", experiment.getCode());
- builder.experiment("experiment_type_code", experiment.getExperimentType().getCode());
- builder.experiment("registration_timestamp", experiment.getRegistrationDate());
- builder.experiment("registrator", experiment.getRegistrator());
- builder.experimentProperties(experiment.getProperties());
- return builder.toString();
- }
-
private String createRootPath(AbstractExternalData dataSet)
{
Sample sample = dataSet.getSample();
@@ -589,28 +337,6 @@ class UploadingCommand implements IDataSetCommand
+ "/" + (sample == null ? "" : sample.getCode() + "/") + dataSet.getCode();
}
- private void addEntry(ZipOutputStream zipOutputStream, String zipEntryPath, long lastModified,
- InputStream in) throws IOException
- {
- try
- {
- ZipEntry zipEntry = new ZipEntry(zipEntryPath.replace('\\', '/'));
- zipEntry.setTime(lastModified);
- zipEntry.setMethod(ZipEntry.DEFLATED);
- zipOutputStream.putNextEntry(zipEntry);
- int len;
- byte[] buffer = new byte[1024];
- while ((len = in.read(buffer)) > 0)
- {
- zipOutputStream.write(buffer, 0, len);
- }
- } finally
- {
- IOUtils.closeQuietly(in);
- zipOutputStream.closeEntry();
- }
- }
-
private void sendEMail(String message)
{
final IMailClient mailClient = new MailClient(mailClientParameters);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ZipDataSetPackager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ZipDataSetPackager.java
new file mode 100644
index 00000000000..e2972b4bfda
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ZipDataSetPackager.java
@@ -0,0 +1,81 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.io.IOUtils;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
+
+import de.schlichtherle.util.zip.ZipEntry;
+import de.schlichtherle.util.zip.ZipOutputStream;
+
+/**
+ * Packager based on a {@link java.util.zip.ZipOutputStream}.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class ZipDataSetPackager extends AbstractDataSetPackager
+{
+ private final ZipOutputStream zipOutputStream;
+ private final boolean compress;
+
+ protected ZipDataSetPackager(ZipOutputStream zipOutputStream, boolean compress, ISimpleLogger logger,
+ IHierarchicalContentProvider contentProvider, DataSetExistenceChecker dataSetExistenceChecker)
+ {
+ super(logger, contentProvider, dataSetExistenceChecker);
+ this.zipOutputStream = zipOutputStream;
+ this.compress = compress;
+ }
+
+ @Override
+ public void addEntry(String entryPath, long lastModified, InputStream in)
+ {
+ try
+ {
+ ZipEntry zipEntry = new ZipEntry(entryPath.replace('\\', '/'));
+ zipEntry.setTime(lastModified);
+ zipEntry.setMethod(compress ? ZipEntry.DEFLATED : ZipEntry.STORED);
+ zipOutputStream.putNextEntry(zipEntry);
+ int len;
+ byte[] buffer = new byte[1024];
+ while ((len = in.read(buffer)) > 0)
+ {
+ zipOutputStream.write(buffer, 0, len);
+ }
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ } finally
+ {
+ IOUtils.closeQuietly(in);
+ try
+ {
+ zipOutputStream.closeEntry();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
index ac357cd6b19..443ec1725f0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/AbstractArchiverProcessingPlugin.java
@@ -541,7 +541,7 @@ public abstract class AbstractArchiverProcessingPlugin extends AbstractDatastore
return shareFinder;
}
- private IEncapsulatedOpenBISService getService()
+ protected IEncapsulatedOpenBISService getService()
{
if (service == null)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedArchiver.java
new file mode 100644
index 00000000000..ccb73e0e747
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedArchiver.java
@@ -0,0 +1,86 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.filesystem.BooleanStatus;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class DistributedArchiver extends AbstractArchiverProcessingPlugin
+{
+ private static final long serialVersionUID = 1L;
+
+ public DistributedArchiver(Properties properties, File storeRoot)
+ {
+ super(properties, storeRoot, null, null);
+ }
+
+ @Override
+ protected DatasetProcessingStatuses doArchive(List<DatasetDescription> datasets, ArchiverTaskContext context)
+ {
+ List<String> dataSetCodes = new ArrayList<String>();
+ for (DatasetDescription datasetDescription : datasets)
+ {
+ dataSetCodes.add(datasetDescription.getDataSetCode());
+ }
+ List<AbstractExternalData> dataSets = getService().listDataSetsByCode(dataSetCodes);
+ return null;
+ }
+
+ @Override
+ protected DatasetProcessingStatuses doUnarchive(List<DatasetDescription> datasets, ArchiverTaskContext context)
+ {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ protected DatasetProcessingStatuses doDeleteFromArchive(List<? extends IDatasetLocation> datasets)
+ {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ protected BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset, ArchiverTaskContext context)
+ {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+ @Override
+ protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset)
+ {
+ // TODO Auto-generated method stub
+ return null;
+ }
+
+}
--
GitLab