diff --git a/common/source/java/ch/systemsx/cisd/common/logging/Log4jSimpleLogger.java b/common/source/java/ch/systemsx/cisd/common/logging/Log4jSimpleLogger.java
index 4466a655fd7952a499a889115b5ce1246b47bd73..a650f0f0d0f5348983c731f0e8a3872c1e4a8a41 100644
--- a/common/source/java/ch/systemsx/cisd/common/logging/Log4jSimpleLogger.java
+++ b/common/source/java/ch/systemsx/cisd/common/logging/Log4jSimpleLogger.java
@@ -92,10 +92,10 @@ public class Log4jSimpleLogger implements ISimpleLogger
{
if (log4jOverridePriorityOrNull != null)
{
- log4jLogger.log(log4jOverridePriorityOrNull, message);
+ log4jLogger.log(log4jOverridePriorityOrNull, message, throwableOrNull);
} else
{
- log4jLogger.log(toLog4jPriority(level), message, null);
+ log4jLogger.log(toLog4jPriority(level), message, throwableOrNull);
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AbstractDataSetPackager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AbstractDataSetPackager.java
index db88e82edcea5c5e4438171519c9e3f5421ece7d..27fa640ee665ebbf65031ed906847bf500b6a1c4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AbstractDataSetPackager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/AbstractDataSetPackager.java
@@ -20,8 +20,6 @@ import java.io.ByteArrayInputStream;
import java.io.InputStream;
import java.util.List;
-import ch.systemsx.cisd.common.logging.ISimpleLogger;
-import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
@@ -38,36 +36,41 @@ public abstract class AbstractDataSetPackager
{
public static final String META_DATA_FILE_NAME = "meta-data.tsv";
- private final ISimpleLogger logger;
private final IHierarchicalContentProvider contentProvider;
private final DataSetExistenceChecker dataSetExistenceChecker;
- protected AbstractDataSetPackager(ISimpleLogger logger, IHierarchicalContentProvider contentProvider,
+ protected AbstractDataSetPackager(IHierarchicalContentProvider contentProvider,
DataSetExistenceChecker dataSetExistenceChecker)
{
- this.logger = logger;
this.contentProvider = contentProvider;
this.dataSetExistenceChecker = dataSetExistenceChecker;
}
+
+ /**
+ * Adds an entry with specified entry path and last modification date filled with data from
+ * specified input stream.
+ */
+ public abstract void addEntry(String entryPath, long lastModified, InputStream in);
+
+ /**
+ * Closes the package.
+ */
+ public abstract void close();
- public boolean addDataSetTo(String rootPath, AbstractExternalData externalData)
+ public void addDataSetTo(String rootPath, AbstractExternalData externalData)
{
try
{
- addEntry(rootPath + META_DATA_FILE_NAME,
- System.currentTimeMillis(),
+ addEntry(rootPath + META_DATA_FILE_NAME, System.currentTimeMillis(),
new ByteArrayInputStream(MetaDataBuilder.createMetaData(externalData).getBytes()));
} catch (Exception ex)
{
- logger.log(LogLevel.ERROR,
- "Couldn't add meta data for data set '" + externalData.getCode()
- + "' to zip file.", ex);
- return false;
+ throw new RuntimeException(
+ "Couldn't package meta data for data set '" + externalData.getCode() + "'.", ex);
}
- if (dataSetExistenceChecker.dataSetExists(DataSetTranslator
- .translateToDescription(externalData)) == false)
+ if (dataSetExistenceChecker.dataSetExists(DataSetTranslator.translateToDescription(externalData)) == false)
{
- return handleNonExistingDataSet(externalData, null);
+ throw handleNonExistingDataSet(externalData, null);
}
IHierarchicalContent root = null;
try
@@ -75,17 +78,14 @@ public abstract class AbstractDataSetPackager
root = contentProvider.asContent(externalData.getCode());
} catch (Exception ex)
{
- return handleNonExistingDataSet(externalData, ex);
+ throw handleNonExistingDataSet(externalData, ex);
}
try
{
addTo(rootPath, root.getRootNode());
- return true;
} catch (Exception ex)
{
- logger.log(LogLevel.ERROR, "Couldn't add data set '" + externalData.getCode()
- + "' to zip file.", ex);
- return false;
+ throw new RuntimeException("Couldn't package data set '" + externalData.getCode() + "'.", ex);
} finally
{
if (root != null)
@@ -95,10 +95,9 @@ public abstract class AbstractDataSetPackager
}
}
- private boolean handleNonExistingDataSet(AbstractExternalData externalData, Exception ex)
+ private RuntimeException handleNonExistingDataSet(AbstractExternalData externalData, Exception ex)
{
- logger.log(LogLevel.ERROR, "Data set " + externalData.getCode() + " does not exist.", ex);
- return false;
+ return new RuntimeException("Data set '" + externalData.getCode() + "' does not exist.", ex);
}
private void addTo(String newRootPath, IHierarchicalContentNode node)
@@ -116,6 +115,4 @@ public abstract class AbstractDataSetPackager
node.getInputStream());
}
}
-
- public abstract void addEntry(String entryPath, long lastModified, InputStream in);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java
index a4a2d0106a6d01b942d349dbe23db3a802feadbc..0adce36e2a7005df0012e6dec7ad75fe496f802d 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommand.java
@@ -17,9 +17,6 @@
package ch.systemsx.cisd.openbis.dss.generic.server;
import java.io.File;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -34,7 +31,6 @@ import ch.systemsx.cisd.cifex.rpc.client.ICIFEXComponent;
import ch.systemsx.cisd.cifex.rpc.client.ICIFEXUploader;
import ch.systemsx.cisd.cifex.rpc.client.gui.IProgressListener;
import ch.systemsx.cisd.cifex.shared.basic.Constants;
-import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.mail.IMailClient;
@@ -52,8 +48,6 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetUploadContext;
-import de.schlichtherle.util.zip.ZipOutputStream;
-
/**
* A command which zips the given data sets and uploads the ZIP file to CIFEX.
*
@@ -278,24 +272,24 @@ class UploadingCommand implements IDataSetCommand
private boolean fillZipFile(IDataSetDirectoryProvider dataSetDirectoryProvider, File zipFile)
{
- OutputStream outputStream = null;
- ZipOutputStream zipOutputStream = null;
+ AbstractDataSetPackager packager = null;
+ DataSetExistenceChecker dataSetExistenceChecker =
+ new DataSetExistenceChecker(dataSetDirectoryProvider,
+ TimingParameters.create(new Properties()));
try
{
- outputStream = new FileOutputStream(zipFile);
- zipOutputStream = new ZipOutputStream(outputStream);
- DataSetExistenceChecker dataSetExistenceChecker =
- new DataSetExistenceChecker(dataSetDirectoryProvider,
- TimingParameters.create(new Properties()));
- Log4jSimpleLogger logger = new Log4jSimpleLogger(notificationLog);
- ZipDataSetPackager packager = new ZipDataSetPackager(zipOutputStream, true,
- logger, getHierarchicalContentProvider(), dataSetExistenceChecker);
+ packager = new ZipDataSetPackager(zipFile, true,
+ getHierarchicalContentProvider(), dataSetExistenceChecker);
for (AbstractExternalData externalData : dataSets)
{
String newRootPath = createRootPath(externalData) + "/";
- boolean success = packager.addDataSetTo(newRootPath, externalData);
- if (success == false)
+ try
+ {
+
+ packager.addDataSetTo(newRootPath, externalData);
+ } catch (RuntimeException ex)
{
+ notificationLog.error(ex.getMessage(), ex);
return false;
}
}
@@ -306,14 +300,14 @@ class UploadingCommand implements IDataSetCommand
return false;
} finally
{
- if (zipOutputStream != null)
+ if (packager != null)
{
try
{
- zipOutputStream.close();
- } catch (IOException ex)
+ packager.close();
+ } catch (Exception ex)
{
- notificationLog.error("Couldn't close zip file", ex);
+ notificationLog.error("Couldn't close package", ex);
}
}
}
@@ -340,8 +334,7 @@ class UploadingCommand implements IDataSetCommand
private void sendEMail(String message)
{
final IMailClient mailClient = new MailClient(mailClientParameters);
- mailClient
- .sendMessage("[Data Set Server] Uploading failed", message, null, null, userEMail);
+ mailClient.sendMessage("[Data Set Server] Uploading failed", message, null, null, userEMail);
}
@Override
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ZipDataSetPackager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ZipDataSetPackager.java
index e2972b4bfda7ca56933e52f216cd60f5276a0f54..a9088d310f07f04560404d39d1894a282e634f87 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ZipDataSetPackager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/ZipDataSetPackager.java
@@ -16,13 +16,14 @@
package ch.systemsx.cisd.openbis.dss.generic.server;
+import java.io.File;
+import java.io.FileOutputStream;
import java.io.IOException;
import java.io.InputStream;
import org.apache.commons.io.IOUtils;
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
-import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
@@ -36,14 +37,16 @@ import de.schlichtherle.util.zip.ZipOutputStream;
*/
public class ZipDataSetPackager extends AbstractDataSetPackager
{
- private final ZipOutputStream zipOutputStream;
+ private final File zipFile;
private final boolean compress;
+
+ private ZipOutputStream zipOutputStream;
- protected ZipDataSetPackager(ZipOutputStream zipOutputStream, boolean compress, ISimpleLogger logger,
+ public ZipDataSetPackager(File zipFile, boolean compress,
IHierarchicalContentProvider contentProvider, DataSetExistenceChecker dataSetExistenceChecker)
{
- super(logger, contentProvider, dataSetExistenceChecker);
- this.zipOutputStream = zipOutputStream;
+ super(contentProvider, dataSetExistenceChecker);
+ this.zipFile = zipFile;
this.compress = compress;
}
@@ -55,12 +58,12 @@ public class ZipDataSetPackager extends AbstractDataSetPackager
ZipEntry zipEntry = new ZipEntry(entryPath.replace('\\', '/'));
zipEntry.setTime(lastModified);
zipEntry.setMethod(compress ? ZipEntry.DEFLATED : ZipEntry.STORED);
- zipOutputStream.putNextEntry(zipEntry);
+ getZipOutputStream().putNextEntry(zipEntry);
int len;
byte[] buffer = new byte[1024];
while ((len = in.read(buffer)) > 0)
{
- zipOutputStream.write(buffer, 0, len);
+ getZipOutputStream().write(buffer, 0, len);
}
} catch (IOException ex)
{
@@ -70,7 +73,7 @@ public class ZipDataSetPackager extends AbstractDataSetPackager
IOUtils.closeQuietly(in);
try
{
- zipOutputStream.closeEntry();
+ getZipOutputStream().closeEntry();
} catch (IOException ex)
{
throw CheckedExceptionTunnel.wrapIfNecessary(ex);
@@ -78,4 +81,37 @@ public class ZipDataSetPackager extends AbstractDataSetPackager
}
}
+ @Override
+ public void close()
+ {
+ if (zipOutputStream != null)
+ {
+ try
+ {
+ zipOutputStream.close();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+ }
+
+ private ZipOutputStream getZipOutputStream()
+ {
+ if (zipOutputStream == null)
+ {
+ FileOutputStream outputStream = null;
+ try
+ {
+ outputStream = new FileOutputStream(zipFile);
+ zipOutputStream = new ZipOutputStream(outputStream);
+ } catch (Exception ex)
+ {
+ IOUtils.closeQuietly(outputStream);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+ return zipOutputStream;
+ }
+
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
index cfe9d747af8b2d37e627a292bb02968b57a198b5..156f9f51eeb5ade68d99cc048b9ae4dde73ec2fd 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DataSetFileOperationsManager.java
@@ -35,6 +35,8 @@ import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.properties.PropertyUtils;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.DefaultFileBasedHierarchicalContentFactory;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.openbis.dss.generic.server.IDataSetFileOperationsExecutor;
import ch.systemsx.cisd.openbis.dss.generic.server.LocalDataSetFileOperationsExcecutor;
import ch.systemsx.cisd.openbis.dss.generic.server.RemoteDataSetFileOperationsExecutor;
@@ -353,8 +355,10 @@ public class DataSetFileOperationsManager implements IDataSetFileOperationsManag
}
@Override
- public File getDestinationFile(DatasetDescription dataset)
+ public IHierarchicalContent getAsHierarchicalContent(DatasetDescription dataset)
{
- return new File(destination, dataset.getDataSetLocation());
+ return new DefaultFileBasedHierarchicalContentFactory()
+ .asHierarchicalContent(new File(destination, dataset.getDataSetLocation()), null);
}
+
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedArchiver.java
index ccb73e0e74785bc940465b4245ba1c1932cdcb8b..55d3b57d22890111009bebe007fb8d91fd39899d 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedArchiver.java
@@ -21,13 +21,23 @@ import java.util.ArrayList;
import java.util.List;
import java.util.Properties;
+import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
+import ch.systemsx.cisd.common.properties.PropertyUtils;
+import ch.systemsx.cisd.common.time.TimingParameters;
+import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDataSetPackager;
+import ch.systemsx.cisd.openbis.dss.generic.server.ZipDataSetPackager;
import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
-import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifierFactory;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFactory;
/**
*
@@ -38,21 +48,91 @@ public class DistributedArchiver extends AbstractArchiverProcessingPlugin
{
private static final long serialVersionUID = 1L;
+ private final boolean compress;
+
+ private File destination;
+
public DistributedArchiver(Properties properties, File storeRoot)
{
super(properties, storeRoot, null, null);
+ destination = new File(properties.getProperty("destination"));
+ compress = PropertyUtils.getBoolean(properties, "compress", true);
+ destination.mkdirs();
}
@Override
protected DatasetProcessingStatuses doArchive(List<DatasetDescription> datasets, ArchiverTaskContext context)
{
- List<String> dataSetCodes = new ArrayList<String>();
+ List<AbstractExternalData> dataSets = getDataSetMetaData(datasets);
+ IHierarchicalContentProvider contentProvider = context.getHierarchicalContentProvider();
+ IDataSetDirectoryProvider directoryProvider = context.getDirectoryProvider();
+ IShareIdManager shareIdManager = directoryProvider.getShareIdManager();
+ DataSetExistenceChecker dataSetExistenceChecker =
+ new DataSetExistenceChecker(directoryProvider, TimingParameters.create(new Properties()));
+ DatasetProcessingStatuses statuses = new DatasetProcessingStatuses();
+ for (AbstractExternalData dataSet : dataSets)
+ {
+ Status status = Status.OK;
+ String dataSetCode = dataSet.getCode();
+ File file = new File(getArchive(dataSet), dataSetCode + ".zip");
+ shareIdManager.lock(dataSetCode);
+ AbstractDataSetPackager dataSetPackager = null;
+ try
+ {
+ dataSetPackager = createPackager(file, contentProvider, dataSetExistenceChecker);
+ dataSetPackager.addDataSetTo("", dataSet);
+ } catch (Exception ex)
+ {
+ status = Status.createError(ex.toString());
+ operationLog.error("Couldn't create package file: " + file, ex);
+ } finally
+ {
+ if (dataSetPackager != null)
+ {
+ try
+ {
+ dataSetPackager.close();
+ } catch (Exception ex)
+ {
+ status = Status.createError("Couldn't close package file: " + file + ": " + ex);
+ }
+ }
+ shareIdManager.releaseLock(dataSetCode);
+ operationLog.info("Data set " + dataSetCode + " archived: " + file);
+ }
+ statuses.addResult(dataSetCode, status, Operation.ARCHIVE);
+ }
+ return statuses;
+ }
+
+ private AbstractDataSetPackager createPackager(File file, IHierarchicalContentProvider contentProvider,
+ DataSetExistenceChecker dataSetExistenceChecker)
+ {
+ return new ZipDataSetPackager(file, compress, contentProvider, dataSetExistenceChecker);
+ }
+
+ private List<AbstractExternalData> getDataSetMetaData(List<DatasetDescription> datasets)
+ {
+ IEncapsulatedOpenBISService service = getService();
+ List<AbstractExternalData> dataSets = new ArrayList<AbstractExternalData>();
for (DatasetDescription datasetDescription : datasets)
{
- dataSetCodes.add(datasetDescription.getDataSetCode());
+ AbstractExternalData dataSet = service.tryGetDataSet(datasetDescription.getDataSetCode());
+ String experimentIdentifier = datasetDescription.getExperimentIdentifier();
+ dataSet.setExperiment(service.tryGetExperiment(ExperimentIdentifierFactory.parse(experimentIdentifier)));
+ String sampleIdentifier = datasetDescription.getSampleIdentifier();
+ if (sampleIdentifier != null)
+ {
+ dataSet.setSample(service.tryGetSampleWithExperiment(SampleIdentifierFactory.parse(sampleIdentifier)));
+ }
+ dataSets.add(dataSet);
}
- List<AbstractExternalData> dataSets = getService().listDataSetsByCode(dataSetCodes);
- return null;
+ return dataSets;
+ }
+
+ private File getArchive(AbstractExternalData dataSet)
+ {
+ return destination;
}
@Override
@@ -72,15 +152,13 @@ public class DistributedArchiver extends AbstractArchiverProcessingPlugin
@Override
protected BooleanStatus isDataSetSynchronizedWithArchive(DatasetDescription dataset, ArchiverTaskContext context)
{
- // TODO Auto-generated method stub
- return null;
+ return BooleanStatus.createFalse();
}
@Override
protected BooleanStatus isDataSetPresentInArchive(DatasetDescription dataset)
{
- // TODO Auto-generated method stub
- return null;
+ return BooleanStatus.createFalse();
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedPackagingDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedPackagingDataSetFileOperationsManager.java
new file mode 100644
index 0000000000000000000000000000000000000000..983e5e59a89efb13441193d163242c47c93fb7b8
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/DistributedPackagingDataSetFileOperationsManager.java
@@ -0,0 +1,363 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.BufferedInputStream;
+import java.io.BufferedOutputStream;
+import java.io.File;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.Collection;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Properties;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.filesystem.BooleanStatus;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.properties.PropertyUtils;
+import ch.systemsx.cisd.common.time.TimingParameters;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.FilteredHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.IHierarchicalContentNodeFilter;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.ZipBasedHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDataSetPackager;
+import ch.systemsx.cisd.openbis.dss.generic.server.ZipDataSetPackager;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DataSetExistenceChecker;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
+import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifierFactory;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifierFactory;
+
+import de.schlichtherle.io.rof.SimpleReadOnlyFile;
+import de.schlichtherle.util.zip.BasicZipFile;
+import de.schlichtherle.util.zip.ZipEntry;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class DistributedPackagingDataSetFileOperationsManager implements IDataSetFileOperationsManager
+{
+ static final String MAPPING_FILE_KEY = "mapping-file";
+
+ static final String CREATE_ARCHIVES_KEY = MAPPING_FILE_KEY + ".create-archives";
+
+ static final String DEFAULT_DESTINATION_KEY = "default-archive-folder";
+
+ static final String WITH_SHARDING_KEY = "with-sharding";
+
+ static final String COMPRESS_KEY = "compressing";
+
+ private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, DistributedPackagingDataSetFileOperationsManager.class);
+
+ private static final IHierarchicalContentNodeFilter FILTER = new IHierarchicalContentNodeFilter()
+ {
+ @Override
+ public boolean accept(IHierarchicalContentNode node)
+ {
+ return AbstractDataSetPackager.META_DATA_FILE_NAME.equals(node.getRelativePath()) == false;
+ }
+ };
+
+ private Map<String, File> spaceArchiveFolderMapping = new HashMap<String, File>();
+
+ private boolean compress;
+
+ private File defaultFolder;
+
+ private boolean withSharding;
+
+ private transient IEncapsulatedOpenBISService service;
+
+ private transient IHierarchicalContentProvider contentProvider;
+
+ private transient IDataSetDirectoryProvider directoryProvider;
+
+ public DistributedPackagingDataSetFileOperationsManager(Properties properties)
+ {
+ this(properties, null, null, null);
+ }
+
+ DistributedPackagingDataSetFileOperationsManager(Properties properties,
+ IEncapsulatedOpenBISService service, IHierarchicalContentProvider contentProvider,
+ IDataSetDirectoryProvider directoryProvider)
+ {
+ this.service = service;
+ this.contentProvider = contentProvider;
+ this.directoryProvider = directoryProvider;
+ compress = PropertyUtils.getBoolean(properties, COMPRESS_KEY, true);
+ withSharding = PropertyUtils.getBoolean(properties, WITH_SHARDING_KEY, false);
+ defaultFolder = new File(PropertyUtils.getMandatoryProperty(properties, DEFAULT_DESTINATION_KEY));
+ if (defaultFolder.isDirectory() == false)
+ {
+ throw new ConfigurationFailureException("Default archive folder '" + defaultFolder.getPath()
+ + "' doesn't exist or is not a folder.");
+ }
+ String mappingFilePath = properties.getProperty(MAPPING_FILE_KEY);
+ if (mappingFilePath != null)
+ {
+ boolean createArchives = PropertyUtils.getBoolean(properties, CREATE_ARCHIVES_KEY, false);
+ spaceArchiveFolderMapping = new SpaceAttributeMappingManager(mappingFilePath, createArchives).getFoldersMap();
+ }
+ }
+
+ @Override
+ public Status copyToDestination(File originalData, DatasetDescription datasetDescription)
+ {
+ AbstractExternalData dataSet = getDataSetWithAllMetaData(datasetDescription);
+ IShareIdManager shareIdManager = getDirectoryProvider().getShareIdManager();
+ DataSetExistenceChecker dataSetExistenceChecker =
+ new DataSetExistenceChecker(getDirectoryProvider(), TimingParameters.create(new Properties()));
+ Status status = Status.OK;
+ String dataSetCode = datasetDescription.getDataSetCode();
+ File file = getArchiveFile(datasetDescription);
+ shareIdManager.lock(dataSetCode);
+ AbstractDataSetPackager dataSetPackager = null;
+ try
+ {
+ dataSetPackager = createPackager(file, dataSetExistenceChecker);
+ dataSetPackager.addDataSetTo("", dataSet);
+ operationLog.info("Data set '" + dataSetCode + "' archived: " + file);
+ } catch (Exception ex)
+ {
+ status = Status.createError(ex.toString());
+ operationLog.error("Couldn't create package file: " + file, ex);
+ } finally
+ {
+ if (dataSetPackager != null)
+ {
+ try
+ {
+ dataSetPackager.close();
+ } catch (Exception ex)
+ {
+ status = Status.createError("Couldn't close package file: " + file + ": " + ex);
+ }
+ }
+ shareIdManager.releaseLock(dataSetCode);
+ }
+ return status;
+ }
+
+ private AbstractDataSetPackager createPackager(File file, DataSetExistenceChecker dataSetExistenceChecker)
+ {
+ return new ZipDataSetPackager(file, compress, getContentProvider(), dataSetExistenceChecker);
+ }
+
+ @Override
+ public Status retrieveFromDestination(File originalData, DatasetDescription datasetDescription)
+ {
+ File file = getArchiveFile(datasetDescription);
+ BasicZipFile zipFile = null;
+ FileOutputStream fileOutputStream = null;
+ try
+ {
+ zipFile = new BasicZipFile(new SimpleReadOnlyFile(file), "UTF-8", true, false);
+ @SuppressWarnings("unchecked")
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ while (entries.hasMoreElements())
+ {
+ ZipEntry entry = entries.nextElement();
+ File outputFile = new File(originalData, entry.getName());
+ if (entry.isDirectory() == false
+ && AbstractDataSetPackager.META_DATA_FILE_NAME.equals(entry.getName()) == false)
+ {
+ outputFile.getParentFile().mkdirs();
+ InputStream inputStream = new BufferedInputStream(zipFile.getInputStream(entry));
+ fileOutputStream = new FileOutputStream(outputFile);
+ BufferedOutputStream outputStream = new BufferedOutputStream(fileOutputStream);
+ try
+ {
+ IOUtils.copyLarge(inputStream, outputStream);
+ } finally
+ {
+ IOUtils.closeQuietly(inputStream);
+ IOUtils.closeQuietly(outputStream);
+ }
+ }
+ }
+ operationLog.info("Data set '" + datasetDescription.getDataSetCode() + "' unzipped from archive '"
+ + file.getPath() + "' to '" + originalData + "'.");
+ return Status.OK;
+ } catch (Exception ex)
+ {
+ return Status.createError(ex.toString());
+ } finally
+ {
+ if (zipFile != null)
+ {
+ try
+ {
+ zipFile.close();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+ IOUtils.closeQuietly(fileOutputStream);
+ }
+ }
+
+ @Override
+ public Status deleteFromDestination(IDatasetLocation dataset)
+ {
+ File archiveFile = tryFindArchiveFile(dataset);
+ if (archiveFile == null)
+ {
+ operationLog.warn("Archive file for data set '" + dataset.getDataSetCode() + "' no konger exists.");
+ return Status.OK;
+ }
+ boolean success = archiveFile.delete();
+ return success ? Status.OK : Status.createError("Couldn't delete archive file '" + archiveFile + "'.");
+ }
+
+ private File tryFindArchiveFile(IDatasetLocation datasetLocation)
+ {
+ File archiveFile = getArchiveFile(defaultFolder, datasetLocation, false);
+ if (archiveFile.isFile())
+ {
+ return archiveFile;
+ }
+ Collection<File> folders = spaceArchiveFolderMapping.values();
+ for (File folder : folders)
+ {
+ archiveFile = getArchiveFile(folder, datasetLocation, false);
+ if (archiveFile.isFile())
+ {
+ return archiveFile;
+ }
+ }
+ return null;
+ }
+
+ @Override
+ public Status markAsDeleted(IDatasetLocation dataset)
+ {
+ return Status.OK;
+ }
+
+ @Override
+ public BooleanStatus isSynchronizedWithDestination(File originalData, DatasetDescription datasetDescription)
+ {
+ return BooleanStatus.createFalse();
+ }
+
+ @Override
+ public BooleanStatus isPresentInDestination(DatasetDescription datasetDescription)
+ {
+ return BooleanStatus.createFalse();
+ }
+
+ @Override
+ public boolean isHosted()
+ {
+ return false;
+ }
+
+ @Override
+ public IHierarchicalContent getAsHierarchicalContent(DatasetDescription dataset)
+ {
+ return new FilteredHierarchicalContent(new ZipBasedHierarchicalContent(getArchiveFile(dataset)), FILTER);
+ }
+
+ private AbstractExternalData getDataSetWithAllMetaData(DatasetDescription datasetDescription)
+ {
+ AbstractExternalData dataSet = getService().tryGetDataSet(datasetDescription.getDataSetCode());
+ String experimentIdentifier = datasetDescription.getExperimentIdentifier();
+ dataSet.setExperiment(getService().tryGetExperiment(ExperimentIdentifierFactory.parse(experimentIdentifier)));
+ String sampleIdentifier = datasetDescription.getSampleIdentifier();
+ if (sampleIdentifier != null)
+ {
+ dataSet.setSample(getService().tryGetSampleWithExperiment(SampleIdentifierFactory.parse(sampleIdentifier)));
+ }
+ return dataSet;
+ }
+
+ private File getArchiveFile(DatasetDescription datasetDescription)
+ {
+ return getArchiveFile(datasetDescription.getSpaceCode(), datasetDescription, true);
+ }
+
+ private File getArchiveFile(String spaceCode, IDatasetLocation datasetLocation, boolean forWriting)
+ {
+ File folder = spaceArchiveFolderMapping.get(spaceCode);
+ if (folder == null)
+ {
+ folder = defaultFolder;
+ }
+ return getArchiveFile(folder, datasetLocation, forWriting);
+ }
+
+ private File getArchiveFile(File baseFolder, IDatasetLocation datasetLocation, boolean forWriting)
+ {
+ File folder = baseFolder;
+ if (withSharding)
+ {
+ folder = new File(folder, datasetLocation.getDataSetLocation());
+ if (forWriting && folder.exists() == false)
+ {
+ folder.mkdirs();
+ }
+ }
+ return new File(folder, datasetLocation.getDataSetCode() + ".zip");
+ }
+
+ private IEncapsulatedOpenBISService getService()
+ {
+ if (service == null)
+ {
+ service = ServiceProvider.getOpenBISService();
+ }
+ return service;
+ }
+
+ private IHierarchicalContentProvider getContentProvider()
+ {
+ if (contentProvider == null)
+ {
+ contentProvider = ServiceProvider.getHierarchicalContentProvider();
+ }
+ return contentProvider;
+ }
+
+ private IDataSetDirectoryProvider getDirectoryProvider()
+ {
+ if (directoryProvider == null)
+ {
+ directoryProvider = ServiceProvider.getDataStoreService().getDataSetDirectoryProvider();
+ }
+ return directoryProvider;
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java
index adb03a60ec7d3f6f838625d07d984fc6937eef73..65c005649e07ccb20212c0621f973205b51fe80c 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/IDataSetFileOperationsManager.java
@@ -20,6 +20,7 @@ import java.io.File;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IDatasetLocation;
import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription;
@@ -72,7 +73,7 @@ public interface IDataSetFileOperationsManager
public boolean isHosted();
/**
- * @return the dataset file in the destination location
+ * @return the dataset file in the destination location as a hierarchical content.
*/
- public File getDestinationFile(DatasetDescription dataset);
+ public IHierarchicalContent getAsHierarchicalContent(DatasetDescription dataset);
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
index 72fca00e0c3c094e8b7b6b438a5cc257daf4331a..44652fe4835964dc5f12908ae8637214ba9eaba7 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java
@@ -28,6 +28,8 @@ import ch.rinn.restrictions.Private;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.BooleanStatus;
+import ch.systemsx.cisd.common.properties.PropertyParametersUtil;
+import ch.systemsx.cisd.common.properties.PropertyParametersUtil.SectionProperties;
import ch.systemsx.cisd.common.properties.PropertyUtils;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.DefaultFileBasedHierarchicalContentFactory;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
@@ -134,6 +136,19 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
public abstract Status execute(IDataSetFileOperationsManager manager,
IDatasetLocation dataSet);
}
+
+ private static final IDataSetFileOperationsManager createFileOperationsManager(Properties properties)
+ {
+ SectionProperties sectionProperties =
+ PropertyParametersUtil.extractSingleSectionProperties(properties, "distributed", false);
+ Properties props = sectionProperties.getProperties();
+ if (props.isEmpty())
+ {
+ return new DataSetFileOperationsManager(properties,
+ new RsyncArchiveCopierFactory(), new SshCommandExecutorFactory());
+ }
+ return new DistributedPackagingDataSetFileOperationsManager(props);
+ }
private transient IDataSetFileOperationsManager fileOperationsManager;
@@ -143,8 +158,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
public RsyncArchiver(Properties properties, File storeRoot)
{
- this(properties, storeRoot, new DataSetFileOperationsManager(properties,
- new RsyncArchiveCopierFactory(), new SshCommandExecutorFactory()));
+ this(properties, storeRoot, createFileOperationsManager(properties));
}
@Private RsyncArchiver(Properties properties, File storeRoot,
@@ -201,11 +215,7 @@ public class RsyncArchiver extends AbstractArchiverProcessingPlugin
.asHierarchicalContent(temp, null);
} else
{
- archivedContent =
- new DefaultFileBasedHierarchicalContentFactory()
- .asHierarchicalContent(
- fileOperationsManager.getDestinationFile(dataset),
- null);
+ archivedContent = fileOperationsManager.getAsHierarchicalContent(dataset);
}
IHierarchicalContentNode root = content.getRootNode();
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/SpaceAttributeMappingManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/SpaceAttributeMappingManager.java
new file mode 100644
index 0000000000000000000000000000000000000000..0a84942fa5ee5b93d45d93cf6fb4d3715f21eead
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/SpaceAttributeMappingManager.java
@@ -0,0 +1,107 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.File;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import com.csvreader.CsvReader;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+
+/**
+ * Helper class to read and provided space-related attributes.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class SpaceAttributeMappingManager
+{
+ private final Map<String, File> folders = new HashMap<String, File>();
+
+ public SpaceAttributeMappingManager(String mappingFile, boolean createArchives)
+ {
+ CsvReader reader = null;
+ try
+ {
+ reader = new CsvReader(mappingFile, '\t');
+ reader.setSkipEmptyRecords(true);
+ reader.setUseComments(false);
+ reader.setComment('#');
+ reader.setTrimWhitespace(true);
+ boolean success = reader.readHeaders();
+ if (success == false)
+ {
+ throw new IllegalArgumentException("Empty mapping file: " + mappingFile);
+ }
+ while (reader.readRecord())
+ {
+ String[] row = reader.getValues();
+ if (row.length != 3)
+ {
+ throw new IllegalArgumentException("Invalid number of row elements in mapping file '"
+ + mappingFile + "': " + Arrays.asList(row));
+ }
+ String space = row[0].toUpperCase();
+ folders.put(space, getArchiveFolder(space, row, createArchives));
+ }
+ } catch (Exception ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ } finally
+ {
+ if (reader != null)
+ {
+ reader.close();
+ }
+ }
+ }
+
+ private File getArchiveFolder(String space, String[] row, boolean createArchives)
+ {
+ File folder = new File(row[2]);
+ if (createArchives)
+ {
+ if (folder.isFile())
+ {
+ throw new IllegalArgumentException("Archive folder '" + folder + "' is a file.");
+ }
+ if (folder.exists() == false)
+ {
+ boolean success = folder.mkdirs();
+ if (success == false)
+ {
+ throw new IllegalArgumentException("Couldn't create archive folder '" + folder + "'.");
+ }
+ }
+ } else
+ {
+ if (folder.isDirectory() == false)
+ {
+ throw new IllegalArgumentException("Archive folder '" + folder + "' for space "
+ + space + " doesn't exists or is a file.");
+ }
+ }
+ return folder;
+ }
+
+ public Map<String, File> getFoldersMap()
+ {
+ return folders;
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/ArchiverPluginFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/ArchiverPluginFactory.java
index 19556ab6f75fbb9d5a73ff5857ba02cb102bc689..2c2aad6dad288f66e603b8b7465eb2a45af3fff0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/ArchiverPluginFactory.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/ArchiverPluginFactory.java
@@ -54,6 +54,18 @@ public class ArchiverPluginFactory
this.archiverProperties = sectionProperties.getProperties();
}
+ /**
+ * Checks creation of archiver plugin is possible for specified store.
+ */
+ public void check(File storeRoot)
+ {
+ if (isArchiverConfigured() == false)
+ {
+ return;
+ }
+ createInstance(storeRoot);
+ }
+
public boolean isArchiverConfigured()
{
return className != null;
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/PluginTaskInfoProvider.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/PluginTaskInfoProvider.java
index 072d277d4966bc1e238d4958ad37b7f1b1d4148d..7597a248b9814440a819fb50ec21ff2d34fc4ca9 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/PluginTaskInfoProvider.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/tasks/PluginTaskInfoProvider.java
@@ -126,6 +126,7 @@ public class PluginTaskInfoProvider implements IPluginTaskInfoProvider
{
processingPlugins.check(true);
reportingPlugins.check(false);
+ archiverTaskFactory.check(storeRoot);
}
@Override
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommandTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommandTest.java
index 06f9976644ab461f26973042431fdd1e01021af1..f4550881eccc5195b7723347fbf570dd4177d1d0 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommandTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/UploadingCommandTest.java
@@ -432,7 +432,19 @@ public class UploadingCommandTest extends AssertJUnit
logRecorder.resetLogContent();
command.execute(null, directoryProvider);
- assertEquals("no emails expected", false, EMAILS.exists());
+ if (EMAILS.exists())
+ {
+ File[] files = EMAILS.listFiles();
+ if (files != null)
+ {
+ StringBuilder builder = new StringBuilder();
+ for (File file : files)
+ {
+ builder.append("\ne-mail:").append(FileUtilities.loadToString(file).trim());
+ }
+ fail("Unexpected e-mail:" + builder);
+ }
+ }
assertEquals(1, TMP.listFiles().length);
checkZipFileContent(TMP.listFiles()[0]);
assertEquals(INFO_UPLOAD_PREFIX
@@ -535,11 +547,10 @@ public class UploadingCommandTest extends AssertJUnit
command.execute(null, directoryProvider);
checkEmail("Couldn't create zip file");
- assertEquals("WARN OPERATION.DataSetExistenceChecker - Data set '2' no longer exists."
- + OSUtilities.LINE_SEPARATOR +
- "ERROR NOTIFY.UploadingCommand - Data set 2 does not exist."
- + OSUtilities.LINE_SEPARATOR + INFO_MAIL_PREFIX
- + "Sending message from 'a@bc.de' to recipients '[user@bc.de]'",
+ assertEquals("WARN OPERATION.DataSetExistenceChecker - Data set '2' no longer exists." + OSUtilities.LINE_SEPARATOR
+ + "ERROR NOTIFY.UploadingCommand - Data set '2' does not exist." + OSUtilities.LINE_SEPARATOR
+ + "java.lang.RuntimeException: Data set '2' does not exist." + OSUtilities.LINE_SEPARATOR
+ + INFO_MAIL_PREFIX + "Sending message from 'a@bc.de' to recipients '[user@bc.de]'",
getNormalizedLogContent());
context.assertIsSatisfied();
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/SpaceAttributeMappingManagerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/SpaceAttributeMappingManagerTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..bca98a9ea82956f374657a63403965b1add04615
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/SpaceAttributeMappingManagerTest.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard;
+
+import java.io.File;
+import java.util.Map;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class SpaceAttributeMappingManagerTest extends AbstractFileSystemTestCase
+{
+ private File as1;
+ private File as2;
+
+ @BeforeMethod
+ public void prepareTestFiles()
+ {
+ as1 = new File(workingDirectory, "a-s1");
+ as1.mkdir();
+ as2 = new File(workingDirectory, "a-s2");
+ as2.mkdir();
+ }
+
+ @Test
+ public void test()
+ {
+ File mappingFile = new File(workingDirectory, "mapping.txt");
+ FileUtilities.writeToFile(mappingFile, "Space\tLive Share\tArchive Folder\n"
+ + "s1\t2\t" + as1 + "\n"
+ + "s2\t4\t" + as2);
+
+ SpaceAttributeMappingManager mappingManager = new SpaceAttributeMappingManager(mappingFile.getPath(), false);
+
+ Map<String, File> foldersMap = mappingManager.getFoldersMap();
+ assertEquals(as1.getPath(), foldersMap.get("S1").getPath());
+ assertEquals(as2.getPath(), foldersMap.get("S2").getPath());
+ }
+
+}
diff --git a/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/AbstractHierarchicalContent.java b/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/AbstractHierarchicalContent.java
new file mode 100644
index 0000000000000000000000000000000000000000..ad7d1572c0e44900ca0f73eaef491e6392999a84
--- /dev/null
+++ b/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/AbstractHierarchicalContent.java
@@ -0,0 +1,95 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.common.io.hierarchical_content;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.regex.Pattern;
+
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+
+/**
+ * Abstract super class of {@link IHierarchicalContent} classes which implements {@link #listMatchingNodes(String)}
+ * and {@link #listMatchingNodes(String, String)}.
+ *
+ * @author Piotr Buczek
+ * @author Franz-Josef Elmer
+ */
+public abstract class AbstractHierarchicalContent implements IHierarchicalContent
+{
+ @Override
+ public List<IHierarchicalContentNode> listMatchingNodes(final String relativePathPattern)
+ {
+ final IHierarchicalContentNode startingNode = getRootNode();
+ final Pattern compiledPattern = Pattern.compile(relativePathPattern);
+ final IHierarchicalContentNodeFilter relativePathFilter =
+ new IHierarchicalContentNodeFilter()
+ {
+ @Override
+ public boolean accept(IHierarchicalContentNode node)
+ {
+ return compiledPattern.matcher(node.getRelativePath()).matches();
+ }
+ };
+
+ List<IHierarchicalContentNode> result = new ArrayList<IHierarchicalContentNode>();
+ findMatchingNodes(startingNode, relativePathFilter, result);
+ return result;
+ }
+
+ @Override
+ public List<IHierarchicalContentNode> listMatchingNodes(final String startingPath, final String fileNamePattern)
+ {
+ final IHierarchicalContentNode startingNode = getNode(startingPath);
+ final Pattern compiledPattern = Pattern.compile(fileNamePattern);
+ final IHierarchicalContentNodeFilter fileNameFilter = new IHierarchicalContentNodeFilter()
+ {
+ @Override
+ public boolean accept(IHierarchicalContentNode node)
+ {
+ return compiledPattern.matcher(node.getName()).matches();
+ }
+ };
+
+ List<IHierarchicalContentNode> result = new ArrayList<IHierarchicalContentNode>();
+ findMatchingNodes(startingNode, fileNameFilter, result);
+ return result;
+ }
+
+ /**
+ * Recursively browses hierarchical content looking for nodes accepted by given
+ * <code>filter</code> and adding them to <code>result</code> list.
+ */
+ private void findMatchingNodes(IHierarchicalContentNode dirNode,
+ IHierarchicalContentNodeFilter filter, List<IHierarchicalContentNode> result)
+ {
+ assert dirNode.isDirectory() : "expected a directory node, got: " + dirNode;
+ for (IHierarchicalContentNode childNode : dirNode.getChildNodes())
+ {
+ if (filter.accept(childNode))
+ {
+ result.add(childNode);
+ }
+ if (childNode.isDirectory())
+ {
+ findMatchingNodes(childNode, filter, result);
+ }
+ }
+ }
+
+}
\ No newline at end of file
diff --git a/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/DefaultFileBasedHierarchicalContent.java b/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/DefaultFileBasedHierarchicalContent.java
index 4c8026decd84fca3591833166f42fbf46657810e..a29276536e1c1446c2c0cfc318538bf629c18b25 100644
--- a/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/DefaultFileBasedHierarchicalContent.java
+++ b/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/DefaultFileBasedHierarchicalContent.java
@@ -17,9 +17,6 @@
package ch.systemsx.cisd.openbis.common.io.hierarchical_content;
import java.io.File;
-import java.util.ArrayList;
-import java.util.List;
-import java.util.regex.Pattern;
import ch.systemsx.cisd.common.action.IDelegatedAction;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
@@ -34,7 +31,7 @@ import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchical
*
* @author Piotr Buczek
*/
-class DefaultFileBasedHierarchicalContent implements IHierarchicalContent
+class DefaultFileBasedHierarchicalContent extends AbstractHierarchicalContent
{
private final IHierarchicalContentFactory hierarchicalContentFactory;
@@ -129,46 +126,6 @@ class DefaultFileBasedHierarchicalContent implements IHierarchicalContent
return hierarchicalContentFactory.asHierarchicalContentNode(this, file);
}
- @Override
- public List<IHierarchicalContentNode> listMatchingNodes(final String relativePathPattern)
- {
- final IHierarchicalContentNode startingNode = getRootNode();
- final Pattern compiledPattern = Pattern.compile(relativePathPattern);
- final IHierarchicalContentNodeFilter relativePathFilter =
- new IHierarchicalContentNodeFilter()
- {
- @Override
- public boolean accept(IHierarchicalContentNode node)
- {
- return compiledPattern.matcher(node.getRelativePath()).matches();
- }
- };
-
- List<IHierarchicalContentNode> result = new ArrayList<IHierarchicalContentNode>();
- findMatchingNodes(startingNode, relativePathFilter, result);
- return result;
- }
-
- @Override
- public List<IHierarchicalContentNode> listMatchingNodes(final String startingPath,
- final String fileNamePattern)
- {
- final IHierarchicalContentNode startingNode = getNode(startingPath);
- final Pattern compiledPattern = Pattern.compile(fileNamePattern);
- final IHierarchicalContentNodeFilter fileNameFilter = new IHierarchicalContentNodeFilter()
- {
- @Override
- public boolean accept(IHierarchicalContentNode node)
- {
- return compiledPattern.matcher(node.getName()).matches();
- }
- };
-
- List<IHierarchicalContentNode> result = new ArrayList<IHierarchicalContentNode>();
- findMatchingNodes(startingNode, fileNameFilter, result);
- return result;
- }
-
@Override
public void close()
{
@@ -226,25 +183,4 @@ class DefaultFileBasedHierarchicalContent implements IHierarchicalContent
return true;
}
- /**
- * Recursively browses hierarchical content looking for nodes accepted by given
- * <code>filter</code> and adding them to <code>result</code> list.
- */
- private static void findMatchingNodes(IHierarchicalContentNode dirNode,
- IHierarchicalContentNodeFilter filter, List<IHierarchicalContentNode> result)
- {
- assert dirNode.isDirectory() : "expected a directory node, got: " + dirNode;
- for (IHierarchicalContentNode childNode : dirNode.getChildNodes())
- {
- if (filter.accept(childNode))
- {
- result.add(childNode);
- }
- if (childNode.isDirectory())
- {
- findMatchingNodes(childNode, filter, result);
- }
- }
- }
-
}
diff --git a/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/FilteredHierarchicalContent.java b/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/FilteredHierarchicalContent.java
new file mode 100644
index 0000000000000000000000000000000000000000..4177f859bbca92d61b326d2803afbcf73c4646e0
--- /dev/null
+++ b/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/FilteredHierarchicalContent.java
@@ -0,0 +1,193 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.common.io.hierarchical_content;
+
+import java.io.File;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.List;
+
+import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked;
+import ch.systemsx.cisd.base.io.IRandomAccessFile;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class FilteredHierarchicalContent implements IHierarchicalContent
+{
+ private final IHierarchicalContent content;
+ private final IHierarchicalContentNodeFilter filter;
+
+ public FilteredHierarchicalContent(IHierarchicalContent content, IHierarchicalContentNodeFilter filter)
+ {
+ this.content = content;
+ this.filter = filter;
+ }
+
+ @Override
+ public IHierarchicalContentNode getRootNode()
+ {
+ return wrap(content.getRootNode());
+ }
+
+ @Override
+ public IHierarchicalContentNode getNode(String relativePath) throws IllegalArgumentException
+ {
+ return wrap(content.getNode(relativePath));
+ }
+
+ @Override
+ public IHierarchicalContentNode tryGetNode(String relativePath)
+ {
+ return wrap(content.tryGetNode(relativePath));
+ }
+
+ @Override
+ public List<IHierarchicalContentNode> listMatchingNodes(String relativePathPattern)
+ {
+ return wrap(content.listMatchingNodes(relativePathPattern));
+ }
+
+ @Override
+ public List<IHierarchicalContentNode> listMatchingNodes(String startingPath, String fileNamePattern)
+ {
+ return wrap(content.listMatchingNodes(startingPath, fileNamePattern));
+ }
+
+ @Override
+ public void close()
+ {
+ content.close();
+ }
+
+ private List<IHierarchicalContentNode> wrap(List<IHierarchicalContentNode> nodes)
+ {
+ List<IHierarchicalContentNode> wrappedNodes = new ArrayList<IHierarchicalContentNode>(nodes.size());
+ for (IHierarchicalContentNode node : nodes)
+ {
+ if (filter.accept(node))
+ {
+ wrappedNodes.add(wrap(node));
+ }
+ }
+ return wrappedNodes;
+ }
+
+ private IHierarchicalContentNode wrap(IHierarchicalContentNode node)
+ {
+ return node == null ? null : new FilteredHierarchicalContentNode(node);
+ }
+
+ private final class FilteredHierarchicalContentNode implements IHierarchicalContentNode
+ {
+ private final IHierarchicalContentNode node;
+
+ FilteredHierarchicalContentNode(IHierarchicalContentNode node)
+ {
+ this.node = node;
+ }
+
+ @Override
+ public String getName()
+ {
+ return node.getName();
+ }
+
+ @Override
+ public String getRelativePath()
+ {
+ return node.getRelativePath();
+ }
+
+ @Override
+ public String getParentRelativePath()
+ {
+ return node.getParentRelativePath();
+ }
+
+ @Override
+ public boolean exists()
+ {
+ return node.exists();
+ }
+
+ @Override
+ public boolean isDirectory()
+ {
+ return node.isDirectory();
+ }
+
+ @Override
+ public long getLastModified()
+ {
+ return node.getLastModified();
+ }
+
+ @Override
+ public List<IHierarchicalContentNode> getChildNodes() throws UnsupportedOperationException
+ {
+ return wrap(node.getChildNodes());
+ }
+
+ @Override
+ public File getFile() throws UnsupportedOperationException
+ {
+ return node.getFile();
+ }
+
+ @Override
+ public File tryGetFile()
+ {
+ return node.tryGetFile();
+ }
+
+ @Override
+ public long getFileLength() throws UnsupportedOperationException
+ {
+ return node.getFileLength();
+ }
+
+ @Override
+ public int getChecksumCRC32() throws UnsupportedOperationException
+ {
+ return node.getChecksumCRC32();
+ }
+
+ @Override
+ public boolean isChecksumCRC32Precalculated()
+ {
+ return node.isChecksumCRC32Precalculated();
+ }
+
+ @Override
+ public IRandomAccessFile getFileContent() throws UnsupportedOperationException, IOExceptionUnchecked
+ {
+ return node.getFileContent();
+ }
+
+ @Override
+ public InputStream getInputStream() throws UnsupportedOperationException, IOExceptionUnchecked
+ {
+ return node.getInputStream();
+ }
+ }
+
+}
diff --git a/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/ZipBasedHierarchicalContent.java b/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/ZipBasedHierarchicalContent.java
new file mode 100644
index 0000000000000000000000000000000000000000..7d87aebfddf75195511220910791d03ce5749bd9
--- /dev/null
+++ b/openbis-common/source/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/ZipBasedHierarchicalContent.java
@@ -0,0 +1,310 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.common.io.hierarchical_content;
+
+import java.io.File;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Enumeration;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.base.io.IRandomAccessFile;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+
+import de.schlichtherle.io.rof.SimpleReadOnlyFile;
+import de.schlichtherle.util.zip.BasicZipFile;
+import de.schlichtherle.util.zip.ZipEntry;
+
+/**
+ * {@link IHierarchicalContent} implementation for ZIP files.
+ *
+ * @author Franz-Josef Elmer
+ */
+public class ZipBasedHierarchicalContent extends AbstractHierarchicalContent
+{
+ private static final String extractName(String relativePath)
+ {
+ int indexOfLastDelimiter = relativePath.lastIndexOf('/');
+ if (indexOfLastDelimiter < 0)
+ {
+ return relativePath;
+ }
+ return relativePath.substring(indexOfLastDelimiter + 1);
+ }
+
+ private static final class ZipContainerNode extends AbstractHierarchicalDirectoryContentNode
+ {
+ private final String relativePath;
+ private final String name;
+ private final List<IHierarchicalContentNode> children = new ArrayList<IHierarchicalContentNode>();
+
+ ZipContainerNode(String relativePath)
+ {
+ this.relativePath = relativePath;
+ name = extractName(relativePath);
+ }
+
+ @Override
+ public String getName()
+ {
+ return name;
+ }
+
+ @Override
+ public boolean exists()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean isDirectory()
+ {
+ return true;
+ }
+
+ @Override
+ public long getLastModified()
+ {
+ return 0;
+ }
+
+ @Override
+ public File getFile() throws UnsupportedOperationException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public File tryGetFile()
+ {
+ return null;
+ }
+
+ @Override
+ public boolean isChecksumCRC32Precalculated()
+ {
+ return true;
+ }
+
+ @Override
+ protected String doGetRelativePath()
+ {
+ return relativePath;
+ }
+
+ @Override
+ protected List<IHierarchicalContentNode> doGetChildNodes()
+ {
+ return new ArrayList<IHierarchicalContentNode>(children);
+ }
+ }
+
+ private static final class ZipContentNode extends AbstractHierarchicalFileContentNode
+ {
+ private final BasicZipFile zipFile;
+ private final ZipEntry zipEntry;
+ private final String relativePath;
+ private final String name;
+
+ ZipContentNode(BasicZipFile zipFile, ZipEntry zipEntry)
+ {
+ this.zipFile = zipFile;
+ this.zipEntry = zipEntry;
+ relativePath = zipEntry.getName();
+ name = extractName(relativePath);
+ }
+
+ @Override
+ public String getName()
+ {
+ return name;
+ }
+
+ @Override
+ public boolean exists()
+ {
+ return true;
+ }
+
+ @Override
+ public boolean isDirectory()
+ {
+ return false;
+ }
+
+ @Override
+ public long getLastModified()
+ {
+ return zipEntry.getTime();
+ }
+
+ @Override
+ public File getFile() throws UnsupportedOperationException
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public File tryGetFile()
+ {
+ return null;
+ }
+
+ @Override
+ protected String doGetRelativePath()
+ {
+ return relativePath;
+ }
+
+ @Override
+ protected long doGetFileLength()
+ {
+ return zipEntry.getSize();
+ }
+
+ @Override
+ public boolean isChecksumCRC32Precalculated()
+ {
+ return true;
+ }
+
+ @Override
+ protected int doGetChecksumCRC32()
+ {
+ return (int) zipEntry.getCrc();
+ }
+
+ @Override
+ protected IRandomAccessFile doGetFileContent()
+ {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ protected InputStream doGetInputStream()
+ {
+ try
+ {
+ return zipFile.getInputStream(zipEntry);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+ }
+
+ private static final class NodeManager
+ {
+ private final Map<String, ZipContainerNode> containerNodes = new HashMap<String, ZipContainerNode>();
+ private final Map<String, IHierarchicalContentNode> allNodes = new HashMap<String, IHierarchicalContentNode>();
+
+ private ZipContainerNode rootNode;
+
+ void handle(BasicZipFile zipFile, ZipEntry zipEntry)
+ {
+ linkNode(new ZipContentNode(zipFile, zipEntry));
+ }
+
+ private void linkNode(IHierarchicalContentNode contentNode)
+ {
+ allNodes.put(contentNode.getRelativePath(), contentNode);
+ String parentRelativePath = contentNode.getParentRelativePath();
+ ZipContainerNode containerNode = containerNodes.get(parentRelativePath);
+ if (containerNode == null)
+ {
+ containerNode = new ZipContainerNode(parentRelativePath);
+ containerNodes.put(parentRelativePath, containerNode);
+ if (parentRelativePath.length() == 0)
+ {
+ rootNode = containerNode;
+ } else
+ {
+ linkNode(containerNode);
+ }
+ }
+ containerNode.children.add(contentNode);
+ }
+ }
+
+ private final BasicZipFile zipFile;
+ private final IHierarchicalContentNode rootNode;
+ private final Map<String, IHierarchicalContentNode> allNodes;
+
+ public ZipBasedHierarchicalContent(File file)
+ {
+ try
+ {
+ NodeManager nodeManager = new NodeManager();
+ zipFile = new BasicZipFile(new SimpleReadOnlyFile(file), "UTF-8", true, false);
+ @SuppressWarnings("unchecked")
+ Enumeration<? extends ZipEntry> entries = zipFile.entries();
+ while (entries.hasMoreElements())
+ {
+ ZipEntry entry = entries.nextElement();
+ nodeManager.handle(zipFile, entry);
+ }
+ rootNode = nodeManager.rootNode;
+ allNodes = nodeManager.allNodes;
+ } catch (Exception ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ @Override
+ public IHierarchicalContentNode getRootNode()
+ {
+ return rootNode;
+ }
+
+ @Override
+ public IHierarchicalContentNode getNode(String relativePath) throws IllegalArgumentException
+ {
+ IHierarchicalContentNode node = tryGetNode(relativePath);
+ if (node == null)
+ {
+ throw new IllegalArgumentException("Resource '" + relativePath + "' does not exist.");
+ }
+ return node;
+ }
+
+ @Override
+ public IHierarchicalContentNode tryGetNode(String relativePath)
+ {
+ return allNodes.get(relativePath);
+ }
+
+ @Override
+ public void close()
+ {
+ try
+ {
+ zipFile.close();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+
+}
diff --git a/openbis-common/sourceTest/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/ZipBasedHierarchicalContentTest.java b/openbis-common/sourceTest/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/ZipBasedHierarchicalContentTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..ae39f426ceba8941b24be5eb7878fe16d000bbfc
--- /dev/null
+++ b/openbis-common/sourceTest/java/ch/systemsx/cisd/openbis/common/io/hierarchical_content/ZipBasedHierarchicalContentTest.java
@@ -0,0 +1,199 @@
+/*
+ * Copyright 2013 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.common.io.hierarchical_content;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStream;
+import java.util.Arrays;
+import java.util.List;
+import java.util.zip.CRC32;
+import java.util.zip.CheckedInputStream;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.io.output.NullOutputStream;
+import org.apache.commons.lang.StringUtils;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
+
+import de.schlichtherle.util.zip.ZipEntry;
+import de.schlichtherle.util.zip.ZipOutputStream;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class ZipBasedHierarchicalContentTest extends AbstractFileSystemTestCase
+{
+
+ @Test
+ public void test() throws Exception
+ {
+ File dataRoot = new File(workingDirectory, "data");
+ File alpha = new File(dataRoot, "alpha");
+ File beta = new File(alpha, "beta");
+ beta.mkdirs();
+ FileUtilities.writeToFile(new File(beta, "hello.txt"), "hello world!");
+ FileUtilities.writeToFile(new File(alpha, "read.me"), "don't read me!");
+ FileUtilities.writeToFile(new File(dataRoot, "change-log.txt"), "nothing really changed.");
+ File zipFile = new File("data.zip");
+ zip(zipFile, dataRoot);
+
+ IHierarchicalContent content = new ZipBasedHierarchicalContent(zipFile);
+
+ IHierarchicalContentNode rootNode = content.getRootNode();
+ assertDirectoryNode("", "", rootNode);
+ List<IHierarchicalContentNode> childNodes = rootNode.getChildNodes();
+ assertDirectoryNode("alpha", "alpha", childNodes.get(0));
+ List<IHierarchicalContentNode> grandChildNodes = childNodes.get(0).getChildNodes();
+ assertDirectoryNode("alpha/beta", "beta", grandChildNodes.get(0));
+ List<IHierarchicalContentNode> grandGrandChildNodes = grandChildNodes.get(0).getChildNodes();
+ assertFileNode("alpha/beta/hello.txt", "hello.txt", "hello world!", grandGrandChildNodes.get(0));
+ assertEquals(1, grandGrandChildNodes.size());
+ assertFileNode("alpha/read.me", "read.me", "don't read me!", grandChildNodes.get(1));
+ assertEquals(2, grandChildNodes.size());
+ assertFileNode("change-log.txt", "change-log.txt", "nothing really changed.", childNodes.get(1));
+ assertEquals(2, childNodes.size());
+ }
+
+ private void assertDirectoryNode(String expectedPath, String expectedName, IHierarchicalContentNode node)
+ {
+ assertPathAndName(expectedPath, expectedName, node);
+ assertEquals(true, node.isDirectory());
+ }
+
+ private void assertFileNode(String expectedPath, String expectedName, String expectedContent,
+ IHierarchicalContentNode node) throws Exception
+ {
+ assertPathAndName(expectedPath, expectedName, node);
+ assertEquals(false, node.isDirectory());
+ assertEquals(expectedContent.length(), node.getFileLength());
+ ByteArrayOutputStream output = new ByteArrayOutputStream();
+ IOUtils.copy(node.getInputStream(), output);
+ assertEquals(expectedContent, output.toString());
+ CRC32 checksum = new CRC32();
+ CheckedInputStream in = new CheckedInputStream(new ByteArrayInputStream(expectedContent.getBytes()), checksum);
+ IOUtils.copy(in, new NullOutputStream());
+ assertEquals(checksum.getValue(), node.getChecksumCRC32());
+
+ }
+
+ private void assertPathAndName(String expectedPath, String expectedName, IHierarchicalContentNode node)
+ {
+ assertEquals(expectedPath, node.getRelativePath());
+ assertEquals(expectedName, node.getName());
+ String parentRelativePath = node.getParentRelativePath();
+ if (expectedPath.equals(""))
+ {
+ assertEquals(null, parentRelativePath);
+ } else
+ {
+ assertEquals(expectedPath,
+ (StringUtils.isBlank(parentRelativePath) ? "" : parentRelativePath + "/") + node.getName());
+ }
+ }
+
+ private void zip(File zipFile, File folder)
+ {
+ OutputStream outputStream = null;
+ ZipOutputStream zipOutputStream = null;
+ try
+ {
+ outputStream = new FileOutputStream(zipFile);
+ zipOutputStream = new ZipOutputStream(outputStream);
+ zip(zipOutputStream, folder, folder);
+ } catch (Exception ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ } finally
+ {
+ if (zipOutputStream != null)
+ {
+ try
+ {
+ zipOutputStream.close();
+ } catch (Exception ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ } else
+ {
+ IOUtils.closeQuietly(outputStream);
+ }
+ }
+ }
+
+ private void zip(ZipOutputStream zipOutputStream, File rootFile, File file)
+ {
+ if (file.isFile())
+ {
+ zipTo(zipOutputStream, rootFile, file);
+ } else if (file.isDirectory())
+ {
+ File[] files = file.listFiles();
+ Arrays.sort(files);
+ for (File childFile : files)
+ {
+ zip(zipOutputStream, rootFile, childFile);
+ }
+ }
+ }
+
+ private void zipTo(ZipOutputStream zipOutputStream, File rootFile, File file)
+ {
+ long lastModified = file.lastModified();
+ FileInputStream in = null;
+ try
+ {
+ in = new FileInputStream(file);
+ ZipEntry zipEntry = new ZipEntry(FileUtilities.getRelativeFilePath(rootFile, file).replace('\\', '/'));
+ zipEntry.setTime(lastModified);
+ zipEntry.setMethod(ZipEntry.DEFLATED);
+ zipOutputStream.putNextEntry(zipEntry);
+ int len;
+ byte[] buffer = new byte[1024];
+ while ((len = in.read(buffer)) > 0)
+ {
+ zipOutputStream.write(buffer, 0, len);
+ }
+ } catch (Exception ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ } finally
+ {
+ IOUtils.closeQuietly(in);
+ try
+ {
+ zipOutputStream.closeEntry();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+ }
+
+}