diff --git a/common/source/java/ch/systemsx/cisd/common/maintenance/IResourceContendingMaintenanceTask.java b/common/source/java/ch/systemsx/cisd/common/maintenance/IResourceContendingMaintenanceTask.java
new file mode 100644
index 0000000000000000000000000000000000000000..588a0f0c7c30224027a120ecf9e8ad65b58eae76
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/maintenance/IResourceContendingMaintenanceTask.java
@@ -0,0 +1,50 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.maintenance;
+
+import java.util.Properties;
+
+/**
+ * A {@link IMaintenanceTask} that requires an exclusive access to a shared system resource.
+ *
+ * @author Kaloyan Enimanev
+ */
+public interface IResourceContendingMaintenanceTask extends IMaintenanceTask
+{
+
+ /**
+ * Maintenance tasks can run concurrently and some of the tasks can require exclusive access to
+ * a certain system resource (e.g. two tasks altering the contents the same directory).
+ * <p>
+ * Instead of dealing with such concurrency issues on a case-by-case basis, tasks have the
+ * possibility to "reserve" an abstract system resource for the time of their execution. If two
+ * maintenance tasks declare they require the same system resource they are never executed
+ * simultaneously.
+ * <p>
+ * The method is executed only *once* per task instance immediately after
+ * {@link #setUp(String, Properties)}.
+ * <p>
+ * NOTE: Currently, maintenance tasks can only declare they need a single resource. Should we
+ * discover a use case where a task needs to acquire multiple resources, we can change the
+ * interface and implement a more complex synchronization strategy.
+ *
+ * @return the name of a resource that the task "locks" for the time of its execution, or
+ * <code>null</code> if no such resources are needed.
+ */
+ public String getRequiredResourceLock();
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/maintenance/MaintenancePlugin.java b/common/source/java/ch/systemsx/cisd/common/maintenance/MaintenancePlugin.java
index 07fee11d7d5366f2bd1f2cdd89be3a0c86dd854e..e65a3364f85685b91fb8f3e83ecd1e0b91373c6e 100644
--- a/common/source/java/ch/systemsx/cisd/common/maintenance/MaintenancePlugin.java
+++ b/common/source/java/ch/systemsx/cisd/common/maintenance/MaintenancePlugin.java
@@ -1,8 +1,12 @@
package ch.systemsx.cisd.common.maintenance;
import java.util.Date;
+import java.util.Map;
import java.util.Timer;
import java.util.TimerTask;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReentrantLock;
import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
@@ -10,10 +14,15 @@ import ch.systemsx.cisd.common.utilities.ClassUtils;
public class MaintenancePlugin
{
+ private static final Map<String /* resource name */, Lock> resourceLocks =
+ new ConcurrentHashMap<String /* resource name */, Lock>();
+
private final IMaintenanceTask task;
private final MaintenanceTaskParameters parameters;
+ private final String requiredResourceLockName;
+
public MaintenancePlugin(MaintenanceTaskParameters parameters)
{
this.parameters = parameters;
@@ -26,20 +35,23 @@ public class MaintenancePlugin
+ parameters.getClassName() + "'", CheckedExceptionTunnel.unwrapIfNecessary(ex));
}
task.setUp(parameters.getPluginName(), parameters.getProperties());
+
+ if (task instanceof IResourceContendingMaintenanceTask)
+ {
+ requiredResourceLockName =
+ ((IResourceContendingMaintenanceTask) task).getRequiredResourceLock();
+ } else
+ {
+ requiredResourceLockName = null;
+ }
}
public void start()
{
- final String timerThreadName = parameters.getPluginName() + " - Maintenance Plugin";
- final Timer workerTimer = new Timer(timerThreadName);
- TimerTask timerTask = new TimerTask()
- {
- @Override
- public void run()
- {
- task.execute();
- }
- };
+ String timerThreadName = parameters.getPluginName() + " - Maintenance Plugin";
+ Timer workerTimer = new Timer(timerThreadName);
+
+ TimerTask timerTask = new MaintenanceTimerTask();
Date startDate = parameters.getStartDate();
if (parameters.isExecuteOnlyOnce())
{
@@ -49,4 +61,52 @@ public class MaintenancePlugin
workerTimer.schedule(timerTask, startDate, parameters.getIntervalSeconds() * 1000);
}
}
+
+ private class MaintenanceTimerTask extends TimerTask
+ {
+ @Override
+ public void run()
+ {
+ acquireLockIfNecessary();
+ try
+ {
+ task.execute();
+ } finally
+ {
+ releaseLockIfNecessay();
+ }
+
+ }
+
+ private void acquireLockIfNecessary()
+ {
+ if (requiredResourceLockName != null)
+ {
+ getLock(requiredResourceLockName).lock();
+ }
+ }
+
+ private void releaseLockIfNecessay()
+ {
+ if (requiredResourceLockName != null)
+ {
+ getLock(requiredResourceLockName).unlock();
+ }
+ }
+
+ private Lock getLock(String resourceName)
+ {
+ synchronized (resourceLocks)
+ {
+ Lock lock = resourceLocks.get(resourceName);
+ if (lock == null)
+ {
+ lock = new ReentrantLock();
+ resourceLocks.put(resourceName, lock);
+ }
+ return lock;
+ }
+ }
+ }
+
}
\ No newline at end of file
diff --git a/common/source/java/ch/systemsx/cisd/common/maintenance/MaintenanceTaskParameters.java b/common/source/java/ch/systemsx/cisd/common/maintenance/MaintenanceTaskParameters.java
index fff995d9101e7065b6a2ec2e643b96fb38dc757b..e1f7ef7fd4d9db570445f743a613d9d37a3befd3 100644
--- a/common/source/java/ch/systemsx/cisd/common/maintenance/MaintenanceTaskParameters.java
+++ b/common/source/java/ch/systemsx/cisd/common/maintenance/MaintenanceTaskParameters.java
@@ -34,19 +34,19 @@ import ch.systemsx.cisd.common.utilities.PropertyUtils;
*/
public class MaintenanceTaskParameters
{
- private static final String TIME_FORMAT = "HH:mm";
+ static final String TIME_FORMAT = "HH:mm";
- private static final int ONE_DAY_IN_SEC = 60 * 60 * 24;
+ static final int ONE_DAY_IN_SEC = 60 * 60 * 24;
- private static final String CLASS_KEY = "class";
+ static final String CLASS_KEY = "class";
- private static final String INTERVAL_KEY = "interval";
+ static final String INTERVAL_KEY = "interval";
- private static final String START_KEY = "start";
+ static final String START_KEY = "start";
// If true the task will be executed exactly one, interval will be ignored. By default set to
// false.
- private static final String ONE_TIME_EXECUTION_KEY = "execute-only-once";
+ static final String ONE_TIME_EXECUTION_KEY = "execute-only-once";
private final String pluginName;
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/maintenance/MaintenanceTaskUtilsTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/maintenance/MaintenanceTaskUtilsTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..f45a9e29eea08b7ff588b945f66dbf9aa17473d8
--- /dev/null
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/maintenance/MaintenanceTaskUtilsTest.java
@@ -0,0 +1,104 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.maintenance;
+
+import java.util.Properties;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.testng.AssertJUnit;
+import org.testng.annotations.Test;
+
+
+/**
+ * @author Kaloyan Enimanev
+ */
+public class MaintenanceTaskUtilsTest extends AssertJUnit
+{
+ private static final int NUM_CONTENDING_TASKS = 5;
+
+ private static final AtomicBoolean executedParallely = new AtomicBoolean(false);
+
+ private static final CountDownLatch latch = new CountDownLatch(NUM_CONTENDING_TASKS);
+
+ private static final String SHARED_RESOURCE_LOCK = "SHARED_LOCK";
+
+
+ public static class ResourceContendingTask implements IResourceContendingMaintenanceTask
+ {
+ private static final AtomicInteger numberActive = new AtomicInteger(0);
+
+ public void setUp(String pluginName, Properties properties)
+ {
+ }
+
+ public void execute()
+ {
+ if (numberActive.incrementAndGet() > 1)
+ {
+ executedParallely.set(true);
+ }
+ try
+ {
+ // simulate some activity
+ Thread.sleep(300);
+ } catch (InterruptedException ex)
+ {
+ ex.printStackTrace();
+ } finally
+ {
+ numberActive.decrementAndGet();
+ latch.countDown();
+ }
+
+ }
+
+ public String getRequiredResourceLock()
+ {
+ return SHARED_RESOURCE_LOCK;
+ }
+ }
+
+ @Test
+ public void testStartupMaintenancePlugins() throws Exception
+ {
+ MaintenanceTaskParameters[] tasks = new MaintenanceTaskParameters[NUM_CONTENDING_TASKS];
+ for (int i = 0; i < tasks.length; i++)
+ {
+ tasks[i] = createTaskParameters("Task-" + i);
+ }
+
+ MaintenanceTaskUtils.startupMaintenancePlugins(tasks);
+
+ // wait for all maintenance tasks to finish
+ latch.await();
+
+ assertFalse("Tasks competing for the same system resource should "
+ + "not be executed in parallel", executedParallely.get());
+ }
+
+ private MaintenanceTaskParameters createTaskParameters(String pluginName)
+ {
+ Properties props = new Properties();
+ props.put(MaintenanceTaskParameters.CLASS_KEY, ResourceContendingTask.class.getName());
+ props.put(MaintenanceTaskParameters.ONE_TIME_EXECUTION_KEY, true);
+ props.put(MaintenanceTaskParameters.START_KEY, System.currentTimeMillis());
+ return new MaintenanceTaskParameters(props, pluginName);
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/Constants.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/Constants.java
index 09ec8935ee1f5a7b8ecb99a9878c7cc5829d9400..bf2db8e42d487fe94659de85f0e05fca012fcd25 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/Constants.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/Constants.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.etlserver;
+import ch.systemsx.cisd.common.maintenance.IResourceContendingMaintenanceTask;
+
/**
*
*
@@ -27,4 +29,10 @@ public class Constants
public static final String ERROR_MARKER_FILE = "_delete_me_after_correcting_errors";
public static final String USER_LOG_FILE = "error-log.txt";
+ /**
+ * a constant used by {@link IResourceContendingMaintenanceTask}-s needing an exclusive access
+ * to the data store folder.
+ */
+ public static final String DATA_STORE_RESOURCE_NAME = "DataStore";
+
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
index dca595f8eced91ebac600bf2722cf270912ec987..6b24f8e7c22135764f63eb02b830b020fe6f242c 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
@@ -36,7 +36,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
*/
public class DefaultStorageProcessor extends AbstractStorageProcessor
{
- static final String ORIGINAL_DIR = "original";
+ public static final String ORIGINAL_DIR = "original";
static final String NO_RENAME = "Couldn't rename '%s' to '%s'.";
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
index 5f1ca00eabd1e1ce0be066a8bb72acc6e4e3c2f4..89b963c27851ca5f8518bae128c396295c09632b 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
@@ -26,12 +26,16 @@ import java.util.Set;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.filesystem.SoftLinkMaker;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.logging.LogInitializer;
-import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.common.maintenance.IResourceContendingMaintenanceTask;
+import ch.systemsx.cisd.common.utilities.ClassUtils;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.etlserver.Constants;
+import ch.systemsx.cisd.etlserver.DefaultStorageProcessor;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUtil;
@@ -41,13 +45,16 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
* Creates the hierarchical structure of data sets registered in openBIS for given data store.
*
* @author Izabela Adamczyk
+ * @author Kaloyan Enimanev
*/
-public class HierarchicalStorageUpdater implements IMaintenanceTask
+public class HierarchicalStorageUpdater implements IResourceContendingMaintenanceTask
{
public static final String STOREROOT_DIR_KEY = "storeroot-dir";
public static final String HIERARCHY_ROOT_DIR_KEY = "hierarchy-root-dir";
+ public static final String HIERARCHY_LINK_NAMING_STRATEGY = "link-naming-strategy";
+
private static final String REBUILDING_HIERARCHICAL_STORAGE = "Rebuilding hierarchical storage";
private static final Logger operationLog =
@@ -55,9 +62,11 @@ public class HierarchicalStorageUpdater implements IMaintenanceTask
private IEncapsulatedOpenBISService openBISService;
- private String storeRoot;
+ private IHierarchicalStorageLinkNamingStrategy linkNamingStrategy;
+
+ private File storeRoot;
- private String hierarchyRoot;
+ private File hierarchyRoot;
public void setUp(String pluginName, Properties pluginProperties)
{
@@ -65,33 +74,60 @@ public class HierarchicalStorageUpdater implements IMaintenanceTask
// TODO 2010-03-23, Piotr Buczek: pluginProperties contain all needed properties
// There is no need to load service properties once again.
Properties properties = DssPropertyParametersUtil.loadServiceProperties();
- storeRoot = PropertyUtils.getMandatoryProperty(properties, STOREROOT_DIR_KEY);
- hierarchyRoot =
+ String storeRootFileName =
+ PropertyUtils.getMandatoryProperty(properties, STOREROOT_DIR_KEY);
+ String hierarchyRootFileName =
PropertyUtils.getMandatoryProperty(properties, pluginName + "."
+ HIERARCHY_ROOT_DIR_KEY);
+
openBISService = ServiceProvider.getOpenBISService();
- operationLog.info("Plugin initialized with: store root = " + storeRoot
- + ", hierarchy root = " + hierarchyRoot);
+ linkNamingStrategy = createLinkNamingStrategy(properties);
+ storeRoot = new File(storeRootFileName);
+ hierarchyRoot = new File(hierarchyRootFileName);
+
+ operationLog.info("Plugin initialized with: store root = " + storeRootFileName
+ + ", hierarchy root = " + hierarchyRootFileName);
}
public void execute()
{
- rebuildHierarchy(new File(storeRoot), openBISService, new File(hierarchyRoot));
+ rebuildHierarchy();
+ }
+
+ /**
+ * requires an exclusive lock of the data store folder.
+ */
+ public String getRequiredResourceLock()
+ {
+ return Constants.DATA_STORE_RESOURCE_NAME;
+ }
+
+ private IHierarchicalStorageLinkNamingStrategy createLinkNamingStrategy(Properties properties)
+ {
+ String linkNamingStrategyClassName =
+ PropertyUtils.getProperty(properties, HIERARCHY_LINK_NAMING_STRATEGY,
+ TemplateBasedLinkNamingStrategy.class.getName());
+ try
+ {
+ return ClassUtils.create(IHierarchicalStorageLinkNamingStrategy.class,
+ Class.forName(linkNamingStrategyClassName), properties);
+ } catch (ClassNotFoundException ex)
+ {
+ throw ConfigurationFailureException.fromTemplate("Wrong '%s' property: %s",
+ HIERARCHY_LINK_NAMING_STRATEGY, ex.getMessage());
+ }
}
/**
* Refreshes the hierarchy of the data inside hierarchical storage accordingly to the database
* content.
*/
- private static void rebuildHierarchy(File storeRoot,
- IEncapsulatedOpenBISService openBISService, File hierarchyRoot)
+ private void rebuildHierarchy()
{
logInfo(REBUILDING_HIERARCHICAL_STORAGE);
- Collection<SimpleDataSetInformationDTO> dataSets = openBISService.listDataSets();
- Map<String, String> newLinkMappings =
- convertDataToLinkMappings(storeRoot, hierarchyRoot, dataSets);
+ Map<String, String> newLinkMappings = convertDataToLinkMappings();
Set<String> toCreate = new HashSet<String>(newLinkMappings.keySet());
- Set<String> toDelete = DataSetHierarchyHelper.extractPaths(hierarchyRoot);
+ Set<String> toDelete = linkNamingStrategy.extractPaths(hierarchyRoot);
Set<String> dontTouch = intersection(toCreate, toDelete);
toCreate.removeAll(dontTouch);
toDelete.removeAll(dontTouch);
@@ -103,21 +139,53 @@ public class HierarchicalStorageUpdater implements IMaintenanceTask
/**
* Extracts a {@link Map}: (target,source) from a collection of data sets.
*/
- private static Map<String, String> convertDataToLinkMappings(File storeRoot,
- File hierarchyRoot, Collection<SimpleDataSetInformationDTO> dataSets)
+ private Map<String, String> convertDataToLinkMappings()
{
+ Collection<SimpleDataSetInformationDTO> dataSets = openBISService.listDataSets();
Map<String, String> linkMappings = new HashMap<String, String>();
for (SimpleDataSetInformationDTO dataSet : dataSets)
{
File targetFile =
- new File(hierarchyRoot, DataSetHierarchyHelper.createHierarchicalPath(dataSet));
+ new File(hierarchyRoot, linkNamingStrategy.createHierarchicalPath(dataSet));
File share = new File(storeRoot, dataSet.getDataSetShareId());
- File sourceFile = new File(share, dataSet.getDataSetLocation());
- linkMappings.put(targetFile.getAbsolutePath(), sourceFile.getAbsolutePath());
+ File dataSetLocationRoot = new File(share, dataSet.getDataSetLocation());
+ File linkSource = determineLinkSource(dataSetLocationRoot);
+ linkMappings.put(targetFile.getAbsolutePath(), linkSource.getAbsolutePath());
}
return linkMappings;
}
+ /**
+ * Some storage processors keep the originally uploaded data sets in an "original" sub-folder.
+ * We would like to link directly to the originally uploaded content and therefore we build
+ * determine the file/folder which we want to link in the following way
+ * <p>
+ * <li>1. If the dataset location contains a single subdirectory named "original" with a single
+ * sub-item F then we link to F (F can be file or directory).
+ * <li>2. else If the dataset location contains a single sub-item F then we link F.
+ * <li>3. else (the dataset location has multiple children) we link to the data set location
+ */
+ private File determineLinkSource(File sourceRoot)
+ {
+ File[] rootChildren = sourceRoot.listFiles();
+ File result = sourceRoot;
+ if (rootChildren != null && rootChildren.length == 1)
+ {
+ result = rootChildren[0];
+
+ if (rootChildren[0].isDirectory()
+ && DefaultStorageProcessor.ORIGINAL_DIR.equals(rootChildren[0].getName()))
+ {
+ File[] originalChildren = rootChildren[0].listFiles();
+ if (originalChildren != null && originalChildren.length == 1)
+ {
+ result = originalChildren[0];
+ }
+ }
+ }
+ return result;
+ }
+
/**
* Removes from the <code>linkMappings</code> map all the elements with keys not belonging to
* <code>keep</code> set.
@@ -216,5 +284,4 @@ public class HierarchicalStorageUpdater implements IMaintenanceTask
operationLog.info(info);
}
}
-
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IHierarchicalStorageLinkNamingStrategy.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IHierarchicalStorageLinkNamingStrategy.java
new file mode 100644
index 0000000000000000000000000000000000000000..070f40a82c73b2548227fb20132477cbf48c5ab6
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/IHierarchicalStorageLinkNamingStrategy.java
@@ -0,0 +1,42 @@
+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.io.File;
+import java.util.Set;
+
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+
+/**
+ * A naming strategy for symbolic links in hiearchical storages.
+ *
+ * @author Kaloyan Enimanev
+ */
+public interface IHierarchicalStorageLinkNamingStrategy
+{
+
+ /**
+ * For a given {@link SimpleDataSetInformationDTO} creates relevant path part e.g.
+ * <code>Instance_AAA/Group_BBB/Project_CCC...</code>
+ */
+ public String createHierarchicalPath(SimpleDataSetInformationDTO data);
+
+ /**
+ * Returns all data set paths located under <code>root</code>.
+ */
+ public Set<String> extractPaths(File root);
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
index b37eaa60d5b7883bab390cc3019d6e428db58d88..394fc57b685197c1846a7ccb9b396e90de66b2f3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreShufflingTask.java
@@ -38,10 +38,11 @@ import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.logging.LogInitializer;
-import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.common.maintenance.IResourceContendingMaintenanceTask;
import ch.systemsx.cisd.common.utilities.ClassUtils;
import ch.systemsx.cisd.common.utilities.PropertyParametersUtil;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.etlserver.Constants;
import ch.systemsx.cisd.etlserver.ETLDaemon;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IShareIdManager;
@@ -57,7 +58,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
*
* @author Franz-Josef Elmer
*/
-public class SegmentedStoreShufflingTask implements IMaintenanceTask
+public class SegmentedStoreShufflingTask implements IResourceContendingMaintenanceTask
{
private static final ISegmentedStoreShuffling DUMMY_SHUFFLING = new ISegmentedStoreShuffling()
{
@@ -221,4 +222,12 @@ public class SegmentedStoreShufflingTask implements IMaintenanceTask
operationLog.info("Segmented store shuffling finished.");
}
+ /**
+ * requires an exclusive lock of the data store folder.
+ */
+ public String getRequiredResourceLock()
+ {
+ return Constants.DATA_STORE_RESOURCE_NAME;
+ }
+
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java
new file mode 100644
index 0000000000000000000000000000000000000000..1bfeb1e133839e7e0ac70600159729a5ab451846
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategy.java
@@ -0,0 +1,231 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.io.File;
+import java.util.HashSet;
+import java.util.Properties;
+import java.util.Set;
+import java.util.regex.Pattern;
+
+import org.apache.commons.lang.StringUtils;
+
+import ch.rinn.restrictions.Private;
+import ch.systemsx.cisd.common.utilities.ExtendedProperties;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+
+/**
+ * TODO
+ *
+ * @author Kaloyan Enimanev
+ */
+public class TemplateBasedLinkNamingStrategy implements IHierarchicalStorageLinkNamingStrategy
+{
+
+ public static final String DEFAULT_LINK_TEMPLATE =
+ "${space}/${project}/${experiment}/${datasettype}+${sample}+${dataset}";
+
+ private static final String LINKS_TEMPLATE_PROP_NAME =
+ HierarchicalStorageUpdater.HIERARCHY_LINK_NAMING_STRATEGY + ".template";
+
+ private static final String NOT_DIRECTLY_CONNECTED = "NOT_DIRECTLY_CONNECTED";
+
+ private final String linkTemplate;
+
+
+ public TemplateBasedLinkNamingStrategy(String template)
+ {
+ if (template == null)
+ {
+ this.linkTemplate = DEFAULT_LINK_TEMPLATE;
+ } else
+ {
+ this.linkTemplate = template;
+ }
+ }
+
+ public TemplateBasedLinkNamingStrategy(Properties configurationProperties)
+ {
+ this(configurationProperties.getProperty(LINKS_TEMPLATE_PROP_NAME));
+
+ }
+
+ /**
+ * For given {@link SimpleDataSetInformationDTO} creates relevant path part.
+ *
+ * @return Instance_AAA/Group_BBB/Project_CCC...</code>
+ */
+ public String createHierarchicalPath(SimpleDataSetInformationDTO data)
+ {
+ ExtendedProperties props = new ExtendedProperties();
+ for (PathVariable pathElement : PathVariable.values())
+ {
+ String pathElementValue = pathElement.extractValueFromData(data);
+ String pathElementName = pathElement.name().toLowerCase();
+ if (pathElementValue == null)
+ {
+ pathElementValue = StringUtils.EMPTY;
+ }
+ props.put(pathElementName, pathElementValue);
+ }
+
+ props.put("template", linkTemplate);
+ // this will evaluate and replace all variables in the value of the property
+ return props.getProperty("template");
+
+ }
+
+ /**
+ * Creates a {@link Set} of data set paths located inside <code>root</code>.
+ */
+ public Set<String> extractPaths(File root)
+ {
+ HashSet<String> set = new HashSet<String>();
+ Pattern matchingFilesFilter = createMatchingFilesFilter(root);
+ accumulatePaths(set, root, matchingFilesFilter, getNestedDirectoryLevels());
+ return set;
+ }
+
+ private Pattern createMatchingFilesFilter(File root)
+ {
+ // TODO KE: refactor with constants
+ final String allMatcher = "([^" + File.separator + "]*)";
+ ExtendedProperties props = new ExtendedProperties();
+ for (PathVariable var : PathVariable.values())
+ {
+ props.put(var.name().toLowerCase(), allMatcher);
+ }
+
+ props.put("template", linkTemplate);
+ String subPathRegex = props.getProperty("template");
+ return Pattern.compile(root.getAbsolutePath() + File.separator + subPathRegex);
+ }
+
+ private int getNestedDirectoryLevels()
+ {
+ return StringUtils.countMatches(linkTemplate, File.separator);
+ }
+
+ @Private
+ static void accumulatePaths(HashSet<String> paths, File dir, Pattern matchingFilesFilter,
+ int maxNestedLevel)
+ {
+ File[] children = dir.listFiles();
+ if (children != null)
+ {
+ for (File child : children)
+ {
+ if (maxNestedLevel > 0)
+ {
+ accumulatePaths(paths, child, matchingFilesFilter, maxNestedLevel - 1);
+ } else
+ {
+ String absolutePath = child.getAbsolutePath();
+ if (matchingFilesFilter.matcher(absolutePath).matches())
+ {
+ paths.add(absolutePath);
+ }
+ }
+ }
+ }
+ }
+
+ enum PathVariable
+ {
+ Dataset
+ {
+ @Override
+ String extractValueFromData(SimpleDataSetInformationDTO data)
+ {
+ return data.getDataSetCode();
+ }
+
+ },
+
+ DataSetType
+ {
+
+ @Override
+ String extractValueFromData(SimpleDataSetInformationDTO data)
+ {
+ return data.getDataSetType();
+ }
+
+ },
+
+ Sample
+ {
+
+ @Override
+ String extractValueFromData(SimpleDataSetInformationDTO data)
+ {
+ String samplePathElement = data.getSampleCode();
+ if (samplePathElement == null)
+ {
+ samplePathElement = NOT_DIRECTLY_CONNECTED;
+ }
+ return samplePathElement;
+ }
+
+ },
+
+ Experiment
+ {
+
+ @Override
+ String extractValueFromData(SimpleDataSetInformationDTO data)
+ {
+ return data.getExperimentCode();
+ }
+
+ },
+
+ Project
+ {
+ @Override
+ String extractValueFromData(SimpleDataSetInformationDTO data)
+ {
+ return data.getProjectCode();
+ }
+ },
+
+ Space
+ {
+
+ @Override
+ String extractValueFromData(SimpleDataSetInformationDTO data)
+ {
+ return data.getGroupCode();
+ }
+ },
+
+ Instance
+ {
+
+ @Override
+ String extractValueFromData(SimpleDataSetInformationDTO data)
+ {
+ return data.getDatabaseInstanceCode();
+ }
+
+ };
+
+
+ abstract String extractValueFromData(SimpleDataSetInformationDTO data);
+
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/DataSetHierarchyHelperTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/DataSetHierarchyHelperTest.java
deleted file mode 100644
index df94d018e10b5d7a9f95ebc77899e4e67c883476..0000000000000000000000000000000000000000
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/DataSetHierarchyHelperTest.java
+++ /dev/null
@@ -1,134 +0,0 @@
-/*
- * Copyright 2009 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.etlserver.plugins;
-
-import java.io.File;
-import java.util.HashSet;
-
-import org.testng.annotations.Test;
-
-import ch.rinn.restrictions.Friend;
-import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
-import ch.systemsx.cisd.common.filesystem.FileUtilities;
-import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
-
-/**
- * Test cases for {@link DataSetHierarchyHelper}.
- *
- * @author Izabela Adamczyk
- */
-@Friend(toClasses = DataSetHierarchyHelper.class)
-public class DataSetHierarchyHelperTest extends AbstractFileSystemTestCase
-{
-
- private static final String DATASET_PATH =
- "Instance_DB-I/Space_GROUP-G/Project_PROJECT-P/Experiment_EXP-E/DataSetType_TYPE-T/Sample_SAMPLE-S/Dataset_DATASET-D";
-
- private static final String SAMPLE = "SAMPLE-S";
-
- private static final String PROJECT = "PROJECT-P";
-
- private static final String GROUP = "GROUP-G";
-
- private static final String EXPERIMENT = "EXP-E";
-
- private static final String TYPE = "TYPE-T";
-
- private static final String LOCATION = "location/L";
-
- private static final String DATASET = "DATASET-D";
-
- private static final String DATABASE_INSTANCE = "DB-I";
-
- private SimpleDataSetInformationDTO createDataSetInfo()
- {
- SimpleDataSetInformationDTO dsInfo = new SimpleDataSetInformationDTO();
- dsInfo.setDatabaseInstanceCode(DATABASE_INSTANCE);
- dsInfo.setDataSetCode(DATASET);
- dsInfo.setDataSetLocation(LOCATION);
- dsInfo.setDataSetType(TYPE);
- dsInfo.setExperimentCode(EXPERIMENT);
- dsInfo.setGroupCode(GROUP);
- dsInfo.setProjectCode(PROJECT);
- dsInfo.setSampleCode(SAMPLE);
- return dsInfo;
- }
-
- @Test
- public void testCreateDataSetPath() throws Exception
- {
- SimpleDataSetInformationDTO dsInfo = createDataSetInfo();
- String path = DataSetHierarchyHelper.createHierarchicalPath(dsInfo);
- assertEquals(DATASET_PATH, path);
- }
-
- @Test
- public void testEntitySeparator() throws Exception
- {
- assertEquals(DataSetHierarchyHelper.ENTITY_SEPARATOR, "_");
- }
-
- @Test
- public void testExtractValidPair() throws Exception
- {
- String merged = "Instance_DB-I";
- DataSetHierarchyHelper.Pair pair = DataSetHierarchyHelper.tryExtractPair(merged);
- assertNotNull(pair);
- assertEquals(DataSetHierarchyHelper.PathElementKey.Instance, pair.getKey());
- assertEquals("DB-I", pair.getValue());
- }
-
- @Test
- public void testExtractValidPairWithMoreUnderscores() throws Exception
- {
- String merged = "Instance_DB_I_2";
- DataSetHierarchyHelper.Pair pair = DataSetHierarchyHelper.tryExtractPair(merged);
- assertNotNull(pair);
- assertEquals(DataSetHierarchyHelper.PathElementKey.Instance, pair.getKey());
- assertEquals("DB_I_2", pair.getValue());
- }
-
- @Test
- public void testExtractNoPairNoUnderscore() throws Exception
- {
- String merged = "Instance-DB-I";
- DataSetHierarchyHelper.Pair pair = DataSetHierarchyHelper.tryExtractPair(merged);
- assertNull(pair);
- }
-
- @Test
- public void testExtractNoPairUnknown() throws Exception
- {
- String merged = "Instance2_DB-I";
- DataSetHierarchyHelper.Pair pair = DataSetHierarchyHelper.tryExtractPair(merged);
- assertNull(pair);
- }
-
- @Test
- public void testAddPaths() throws Exception
- {
- HashSet<String> set = new HashSet<String>();
- File root = new File(workingDirectory, "root");
- root.mkdirs();
- File dataSetPath = new File(root, DATASET_PATH);
- dataSetPath.mkdirs();
- DataSetHierarchyHelper.addPaths(set, root, DataSetHierarchyHelper.PathElementKey.Instance);
- assertTrue(set.contains(dataSetPath.getAbsolutePath()));
- FileUtilities.deleteRecursively(root);
- assertFalse(root.exists());
- }
-}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..561330eb43ddada5f62593220e3a7488da67760d
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/TemplateBasedLinkNamingStrategyTest.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import static ch.systemsx.cisd.etlserver.plugins.TemplateBasedLinkNamingStrategy.DEFAULT_LINK_TEMPLATE;
+
+import java.io.File;
+import java.util.Set;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+
+/**
+ * Test cases for {@link TemplateBasedLinkNamingStrategy}.
+ *
+ * @author Izabela Adamczyk
+ * @author Kaloyan Enimanev
+ */
+public class TemplateBasedLinkNamingStrategyTest extends AbstractFileSystemTestCase
+{
+
+ public static final String LONG_LINK_TEMPLATE =
+ "Instance_${instance}/Space_${space}/Project_${project}/Experiment_${experiment}/DataSetType_${datasettype}/Sample_${sample}/Dataset_${dataset}";
+
+ private static final String DATASET_PATH_LONG =
+ "Instance_DB-I/Space_GROUP-G/Project_PROJECT-P/Experiment_EXP-E/DataSetType_TYPE-T/Sample_SAMPLE-S/Dataset_DATASET-D";
+
+ private static final String DATASET_PATH_DEFAULT =
+ "GROUP-G/PROJECT-P/EXP-E/TYPE-T+SAMPLE-S+DATASET-D";
+
+ private static final String SAMPLE = "SAMPLE-S";
+
+ private static final String PROJECT = "PROJECT-P";
+
+ private static final String GROUP = "GROUP-G";
+
+ private static final String EXPERIMENT = "EXP-E";
+
+ private static final String TYPE = "TYPE-T";
+
+ private static final String LOCATION = "location/L";
+
+ private static final String DATASET = "DATASET-D";
+
+ private static final String DATABASE_INSTANCE = "DB-I";
+
+ @Test
+ public void testCreateDataSetPath() throws Exception
+ {
+ assertEquals(DATASET_PATH_DEFAULT, createPathFromTemplate(DEFAULT_LINK_TEMPLATE));
+ assertEquals(DATASET_PATH_LONG, createPathFromTemplate(LONG_LINK_TEMPLATE));
+
+ }
+
+ @Test
+ public void testExtractPathsFromFileSystem() throws Exception
+ {
+ assertFile(DATASET_PATH_DEFAULT).isExtractedWithTemplate(DEFAULT_LINK_TEMPLATE);
+ assertFile(DATASET_PATH_LONG).isExtractedWithTemplate(LONG_LINK_TEMPLATE);
+ }
+
+ private String createPathFromTemplate(String template)
+ {
+ SimpleDataSetInformationDTO dsInfo = createDataSetInfo();
+ return new TemplateBasedLinkNamingStrategy(template)
+ .createHierarchicalPath(dsInfo);
+ }
+
+ private SimpleDataSetInformationDTO createDataSetInfo()
+ {
+ SimpleDataSetInformationDTO dsInfo = new SimpleDataSetInformationDTO();
+ dsInfo.setDatabaseInstanceCode(DATABASE_INSTANCE);
+ dsInfo.setDataSetCode(DATASET);
+ dsInfo.setDataSetLocation(LOCATION);
+ dsInfo.setDataSetType(TYPE);
+ dsInfo.setExperimentCode(EXPERIMENT);
+ dsInfo.setGroupCode(GROUP);
+ dsInfo.setProjectCode(PROJECT);
+ dsInfo.setSampleCode(SAMPLE);
+ return dsInfo;
+ }
+
+ private PathRecognizingAssertions assertFile(String fileName)
+ {
+ return new PathRecognizingAssertions(fileName);
+ }
+
+ private class PathRecognizingAssertions
+ {
+
+ private String fileName;
+
+ public PathRecognizingAssertions(String fileName)
+ {
+ this.fileName = fileName;
+ }
+
+ private void isExtractedWithTemplate(String template)
+ {
+ File dataSetPath = new File(workingDirectory, fileName);
+ dataSetPath.mkdirs();
+
+ TemplateBasedLinkNamingStrategy strategy =
+ new TemplateBasedLinkNamingStrategy(template);
+ Set<String> paths = strategy.extractPaths(workingDirectory);
+ assertTrue(paths.contains(dataSetPath.getAbsolutePath()));
+ }
+
+
+ }
+}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/client/api/v1/impl/DssComponentTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/client/api/v1/impl/DssComponentTest.java
index 84f3cc03955c0f345eb85d03d416b3feb793d7ce..d2a600d8a64eda8502163fb7316127a010339075 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/client/api/v1/impl/DssComponentTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/client/api/v1/impl/DssComponentTest.java
@@ -114,7 +114,6 @@ public class DssComponentTest extends AbstractFileSystemTestCase
//
// AbstractAutoProxyCreator
//
- @SuppressWarnings("rawtypes")
@Override
protected final Object[] getAdvicesAndAdvisorsForBean(final Class beanClass,
final String beanName, final TargetSource customTargetSource)
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/client/web/server/resultset/ColumnSortUtils.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/client/web/server/resultset/ColumnSortUtils.java
index 29879cf98a2bd049abc452754098698e27cb379c..fc6a9a08b508fa13aee982e57e632df0f99cf7d6 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/client/web/server/resultset/ColumnSortUtils.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/client/web/server/resultset/ColumnSortUtils.java
@@ -106,8 +106,8 @@ class ColumnSortUtils
public int compare(GridRowModel<T> o1, GridRowModel<T> o2)
{
- Comparable v1 = sortField.tryGetComparableValue(o1);
- Comparable v2 = sortField.tryGetComparableValue(o2);
+ Comparable<?> v1 = sortField.tryGetComparableValue(o1);
+ Comparable<?> v2 = sortField.tryGetComparableValue(o2);
// treat null as minimal value
if (v1 == null)
{
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/client/web/server/util/FilterUtilsTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/client/web/server/util/FilterUtilsTest.java
index 953ade33e84b3c503f9b374599bb433ec6b16920..99b8af0b217b6a3125b19b2bda22189aabaeef03 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/client/web/server/util/FilterUtilsTest.java
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/client/web/server/util/FilterUtilsTest.java
@@ -86,7 +86,9 @@ public class FilterUtilsTest extends AssertJUnit
@Test
public void test()
{
+ @SuppressWarnings(value = "rawtypes")
CustomFilterInfo<?> filterInfo = new CustomFilterInfo();
+
filterInfo.setExpression("row.col('VALUE') < ${threshold}");
ParameterWithValue parameter = new ParameterWithValue();
parameter.setParameter("threshold");