diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/tar/Untar.java b/common/source/java/ch/systemsx/cisd/common/filesystem/tar/Untar.java
index fc30cba5863a147cb935ef7176e293496805f046..c1713ef37aa07d8f7a7965f31eda4d4bc1a271da 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/tar/Untar.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/tar/Untar.java
@@ -125,13 +125,14 @@ public class Untar implements Closeable
}
}
- public File getEntryInLocation(TarArchiveEntry entry, final Map<String, File> locations)
+ private File getEntryInLocation(TarArchiveEntry entry, final Map<String, File> locations)
{
String[] parts = entry.getName().split("/", 2);
String head = parts[0];
String tail = parts[1];
- return new File(locations.get(head), tail);
+ File parent = locations.get(head);
+ return parent == null ? null : new File(parent, tail);
}
private void extractEntry(TarArchiveEntry entry, final File entryFile, final List<TarArchiveEntry> dirEntries) throws FileNotFoundException,
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
index 4d09159f259dd3717599b232a52525a475649f5a..bb6b32680921001abc6aaa2e6cc5569c3f87e787 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/postregistration/EagerShufflingTask.java
@@ -17,6 +17,7 @@
package ch.systemsx.cisd.etlserver.postregistration;
import java.io.File;
+import java.util.Date;
import java.util.List;
import java.util.Properties;
import java.util.Set;
@@ -60,6 +61,8 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
*/
public class EagerShufflingTask extends AbstractPostRegistrationTaskForPhysicalDataSets
{
+ private static final int SHARES_CACHING_TIMEOUT = 60 * 60 * 1000;
+
@Private
public static final String SHARE_FINDER_KEY = "share-finder";
@@ -176,6 +179,9 @@ public class EagerShufflingTask extends AbstractPostRegistrationTaskForPhysicalD
return null;
}
}
+
+ private List<Share> shares;
+ private Date sharesTimestamp;
@Override
public IPostRegistrationTaskExecutor createExecutor(String dataSetCode)
@@ -183,6 +189,19 @@ public class EagerShufflingTask extends AbstractPostRegistrationTaskForPhysicalD
return new Executor(dataSetCode);
}
+ private List<Share> getShares()
+ {
+ if (shares == null || sharesTimestamp == null
+ || sharesTimestamp.getTime() + SHARES_CACHING_TIMEOUT < System.currentTimeMillis())
+ {
+ shares = SegmentedStoreUtils.getSharesWithDataSets(storeRoot, dataStoreCode,
+ FilterOptions.AVAILABLE_FOR_SHUFFLING,
+ incomingShares, freeSpaceProvider, service, logger);
+ sharesTimestamp = new Date();
+ }
+ return shares;
+ }
+
private final class Executor implements IPostRegistrationTaskExecutor
{
private final String dataSetCode;
@@ -199,17 +218,15 @@ public class EagerShufflingTask extends AbstractPostRegistrationTaskForPhysicalD
@Override
public ICleanupTask createCleanupTask()
{
- List<Share> shares =
- SegmentedStoreUtils.getSharesWithDataSets(storeRoot, dataStoreCode, FilterOptions.AVAILABLE_FOR_SHUFFLING,
- incomingShares, freeSpaceProvider, service, logger);
- dataSet = findDataSet(shares, dataSetCode);
+ List<Share> currentShares = getShares();
+ dataSet = findDataSet(currentShares, dataSetCode);
if (dataSet.getStatus().isAvailable() == false)
{
logger.log(LogLevel.WARN, "Data set " + dataSetCode + " couldn't been shuffled because "
+ "its archiving status is " + dataSet.getStatus());
return new NoCleanupTask();
}
- shareWithMostFreeOrNull = finder.tryToFindShare(dataSet, shares);
+ shareWithMostFreeOrNull = finder.tryToFindShare(dataSet, currentShares);
if (shareWithMostFreeOrNull == null)
{
String message = "No share found for shuffling data set " + dataSetCode + ".";
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TarPackageManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TarPackageManager.java
index 8207d8cbec80a4cb473f9ddc1092b06953968488..8bc177e8da32ee717a71b7cbbd6ad5b12092416b 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TarPackageManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/TarPackageManager.java
@@ -55,7 +55,7 @@ public class TarPackageManager extends AbstractPackageManager
private final int bufferSize;
- private final ISimpleLogger ioSpeedLogger;
+ protected final ISimpleLogger logger;
private Long maxQueueSize;
@@ -65,7 +65,7 @@ public class TarPackageManager extends AbstractPackageManager
bufferSize = PropertyUtils.getInt(properties, BUFFER_SIZE_KEY, DEFAULT_BUFFER_SIZE);
long maxSize = PropertyUtils.getLong(properties, MAXIMUM_QUEUE_SIZE_IN_BYTES_KEY, 5 * bufferSize);
maxQueueSize = maxSize == 0 ? null : maxSize;
- this.ioSpeedLogger = ioSpeedLogger;
+ this.logger = ioSpeedLogger;
}
@Override
@@ -126,7 +126,7 @@ public class TarPackageManager extends AbstractPackageManager
if (onlyMetaData)
{
final ISingleDataSetPathInfoProvider pathInfoProvider
- = new TarBasedPathInfoProvider(packageFile, bufferSize, ioSpeedLogger);
+ = new TarBasedPathInfoProvider(packageFile, bufferSize, logger);
return new PathInfoProviderBasedHierarchicalContent(pathInfoProvider, null, new IDelegatedAction()
{
@Override
@@ -135,7 +135,7 @@ public class TarPackageManager extends AbstractPackageManager
}
});
}
- return new TarBasedHierarchicalContent(packageFile, tempFolder, bufferSize, ioSpeedLogger);
+ return new TarBasedHierarchicalContent(packageFile, tempFolder, bufferSize, logger);
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetArchiver.java
index d2f6469a2fb756fd06d7a79abf539c3fc035127a..af56d49e14684000193ff601ab5f98bc9474785c 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetArchiver.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetArchiver.java
@@ -34,6 +34,7 @@ import org.apache.commons.lang.time.DateUtils;
import ch.rinn.restrictions.Private;
import ch.systemsx.cisd.common.collection.CollectionUtils;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.NotImplementedException;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
@@ -62,6 +63,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.ArchiverTaskContext;
import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSetDirectoryProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IDataStoreServiceInternal;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
import ch.systemsx.cisd.openbis.dss.generic.shared.IUnarchivingPreparation;
import ch.systemsx.cisd.openbis.dss.generic.shared.IncomingShareIdProvider;
@@ -544,16 +546,31 @@ public class MultiDataSetArchiver extends AbstractArchiverProcessingPlugin
@Override
protected DatasetProcessingStatuses doUnarchive(List<DatasetDescription> dataSets, ArchiverTaskContext context)
{
+ DatasetProcessingStatuses result = new DatasetProcessingStatuses();
List<String> dataSetCodes = translateToDataSetCodes(dataSets);
Set<Long> containerIds = assertUnarchivingCapacityNotExceeded(dataSetCodes);
assertNoAvailableDatasets(dataSetCodes);
context.getUnarchivingPreparation().prepareForUnarchiving(dataSets);
+ IMultiDataSetFileOperationsManager operations = getFileOperations();
for (Long containerId : containerIds)
{
MultiDataSetArchiverContainerDTO container = getReadonlyQuery().getContainerForId(containerId);
- getFileOperations().restoreDataSetsFromContainerInFinalDestination(container.getPath(), dataSets);
+ Status status = operations.restoreDataSetsFromContainerInFinalDestination(container.getPath(), dataSets);
+ if (status.isError())
+ {
+ result.addResult(dataSets, status, Operation.UNARCHIVE);
+ return result;
+ }
+ }
+
+ IHierarchicalContentProvider contentProvider = context.getHierarchicalContentProvider();
+ for (String dataSetCode : dataSetCodes)
+ {
+ IHierarchicalContent content = contentProvider.asContentWithoutModifyingAccessTimestamp(dataSetCode);
+ IHierarchicalContentNode rootNode = content.getRootNode();
+ assertFilesExists(dataSetCode, rootNode);
}
for (String dataSetCode : dataSetCodes)
@@ -561,10 +578,45 @@ public class MultiDataSetArchiver extends AbstractArchiverProcessingPlugin
getService().notifyDatasetAccess(dataSetCode);
}
- DatasetProcessingStatuses result = new DatasetProcessingStatuses();
result.addResult(dataSets, Status.OK, Operation.UNARCHIVE);
return result;
}
+
+ private void assertFilesExists(String dataSetCode, IHierarchicalContentNode node)
+ {
+ File file;
+ try
+ {
+ file = node.getFile();
+ } catch (UnsupportedOperationException ex)
+ {
+ throw createException(dataSetCode, node, ex);
+ }
+ if (file.exists() == false)
+ {
+ throw createException(dataSetCode, node, null);
+ }
+ if (node.isDirectory())
+ {
+ for (IHierarchicalContentNode child : node.getChildNodes())
+ {
+ assertFilesExists(dataSetCode, child);
+ }
+ }
+ }
+
+ private EnvironmentFailureException createException(String dataSetCode,
+ IHierarchicalContentNode node, Exception exOrNull)
+ {
+ String message = "Data set " + dataSetCode + ": File '" + node.getRelativePath()
+ + "' does not exist.";
+ if (exOrNull != null)
+ {
+ return new EnvironmentFailureException(message + " (reason: " + exOrNull.getMessage() + ")",
+ exOrNull);
+ }
+ return new EnvironmentFailureException(message);
+ }
private void assertNoAvailableDatasets(List<String> dataSetCodes)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
index 11ae86fc6c1115fe9ce6832c5c6bce46a943d775..b8274759bae5be37526dcc5b8794828826b4c07f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetFileOperationsManager.java
@@ -241,9 +241,7 @@ public class MultiDataSetFileOperationsManager extends AbstractDataSetFileOperat
}
File stageArchiveContainerFile = new File(getFinalArchive().getDestination(), containerPath);
- packageManager.extractMultiDataSets(stageArchiveContainerFile, dataSetToLocation);
-
- return Status.OK;
+ return packageManager.extractMultiDataSets(stageArchiveContainerFile, dataSetToLocation);
}
@Override
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetPackageManager.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetPackageManager.java
index d43e1c552980ae2e045203a274d6fd7baf333a28..974cddfa0c4d97a5ee65af7b1479cdcbad800561 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetPackageManager.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/archiver/MultiDataSetPackageManager.java
@@ -26,6 +26,7 @@ import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.filesystem.tar.Untar;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.openbis.dss.generic.server.AbstractDataSetPackager;
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard.TarPackageManager;
@@ -60,6 +61,7 @@ public class MultiDataSetPackageManager extends TarPackageManager implements IMu
return Status.OK;
} catch (Exception ex)
{
+ logger.log(LogLevel.ERROR, "Error during untaring " + packageFile, ex);
return Status.createError(ex.toString());
} finally
{
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/ExperimentProjectSpaceCodeRecord.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/ExperimentProjectSpaceCodeRecord.java
index 6c244cb537d20182933e9b49440e55ba47be2c5e..581f0c90f6909eb02cdc24e5135d8bb2f27d172b 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/ExperimentProjectSpaceCodeRecord.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/ExperimentProjectSpaceCodeRecord.java
@@ -9,6 +9,8 @@ import ch.rinn.restrictions.Private;
@Private
public class ExperimentProjectSpaceCodeRecord
{
+ public long id;
+
public String e_code;
public String e_permid;
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/ISecondaryEntityListingQuery.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/ISecondaryEntityListingQuery.java
index f9cc0dc247ac0e308ebbee2270e15d14bc04775c..628275f4ad7e6fa2d5598e04ae324ab532a1202b 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/ISecondaryEntityListingQuery.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/ISecondaryEntityListingQuery.java
@@ -37,6 +37,12 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space;
{ SampleReferenceRecord.class })
public interface ISecondaryEntityListingQuery extends BaseQuery
{
+ public static final String SELECT_FROM_EXPERIMENTS
+ = "select e.id as id, e.code as e_code, e.perm_id as e_permid, e.del_id as del_id, et.code as et_code, "
+ + "p.code as p_code, p.id as p_id, p.perm_id as p_perm_id, g.code as spc_code, g.dbin_id as dbin_id from experiments e "
+ + "join experiment_types et on e.exty_id=et.id join projects p on e.proj_id=p.id "
+ + "join spaces g on p.space_id=g.id";
+
public static final int FETCH_SIZE = 1000;
//
@@ -48,12 +54,13 @@ public interface ISecondaryEntityListingQuery extends BaseQuery
*
* @param experimentId The id of the experiment to get the code for.
*/
- @Select("select e.code as e_code, e.perm_id as e_permid, e.del_id as del_id, et.code as et_code, "
- + "p.code as p_code, p.id as p_id, p.perm_id as p_perm_id, g.code as spc_code from experiments e "
- + "join experiment_types et on e.exty_id=et.id join projects p on e.proj_id=p.id "
- + "join spaces g on p.space_id=g.id where e.id=?{1}")
+ @Select(SELECT_FROM_EXPERIMENTS + " where e.id=?{1}")
public ExperimentProjectSpaceCodeRecord getExperimentAndProjectAndGroupCodeForId(
long experimentId);
+
+ @Select(sql = SELECT_FROM_EXPERIMENTS + " where e.id = any(?{1})", parameterBindings =
+ { LongSetMapper.class }, fetchSize = FETCH_SIZE)
+ public DataIterator<ExperimentProjectSpaceCodeRecord> getExperiments(LongSet experimentIds);
//
// Samples
@@ -93,8 +100,6 @@ public interface ISecondaryEntityListingQuery extends BaseQuery
/**
* Returns all spaces of this data base instance.
- *
- * @param databaseInstanceId The id of the database to get the spaces for.
*/
@Select("select id, code from spaces")
public Space[] getAllSpaces();
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/SecondaryEntityDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/SecondaryEntityDAO.java
index 8a2929c3aeedc78e8b9802e04374550c7079036a..944c8d443f89f3001f616d00c93036fe039cd0d7 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/SecondaryEntityDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/common/entity/SecondaryEntityDAO.java
@@ -164,7 +164,18 @@ public class SecondaryEntityDAO
}
return result;
}
-
+
+ public Long2ObjectMap<Experiment> getExperiments(LongSet experimentIds)
+ {
+ Iterable<ExperimentProjectSpaceCodeRecord> experimentRecords = query.getExperiments(experimentIds);
+ Long2ObjectMap<Experiment> result = new Long2ObjectOpenHashMap<Experiment>();
+ for (ExperimentProjectSpaceCodeRecord record : experimentRecords)
+ {
+ result.put(record.id, tryCreateExperiment(record.id, record));
+ }
+ return result;
+ }
+
public LongSet getSampleDescendantIdsAndSelf(Long sampleId)
{
LongSet results = new LongOpenHashSet();
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
index 0da7b1d8f037fc12b96636589050755e351f1af2..cda4425eff61b4ef5059ba57fbe8a243f93028d2 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/datasetlister/DatasetLister.java
@@ -826,11 +826,26 @@ public class DatasetLister extends AbstractLister implements IDatasetLister
return ids;
}
+ private static LongSet extractExperimentIds(Long2ObjectMap<AbstractExternalData> datasetMap)
+ {
+ LongSet ids = new LongOpenHashSet();
+ for (AbstractExternalData dataset : datasetMap.values())
+ {
+ Experiment experiment = dataset.getExperiment();
+ if (experiment != null)
+ {
+ ids.add(experiment.getId());
+ }
+ }
+ return ids;
+ }
+
// assumes that the connection to experiment has been already established and experiment has the
// id set.
private void enrichWithExperiments(Long2ObjectMap<AbstractExternalData> datasetMap)
{
- Long2ObjectMap<Experiment> experimentMap = new Long2ObjectOpenHashMap<Experiment>();
+ LongSet ids = extractExperimentIds(datasetMap);
+ Long2ObjectMap<Experiment> experiments = referencedEntityDAO.getExperiments(ids);
for (AbstractExternalData dataset : datasetMap.values())
{
@@ -839,14 +854,7 @@ public class DatasetLister extends AbstractLister implements IDatasetLister
continue;
}
long experimentId = dataset.getExperiment().getId();
- Experiment experiment = experimentMap.get(experimentId);
- // null value is put if experiment is from different db instance
- if (experimentMap.containsKey(experimentId) == false)
- {
- experiment = referencedEntityDAO.tryGetExperiment(experimentId);
- experimentMap.put(experimentId, experiment);
- }
- dataset.setExperiment(experiment);
+ dataset.setExperiment(experiments.get(experimentId));
}
}