SSDM-1191: Introducing the configurable auto archiving policy GroupingPolicy...

SSDM-1191: Introducing the configurable auto archiving policy GroupingPolicy which should replace ByExperiment and BySpace.

SVN: 33336

datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BaseGroupingPolicy.java

@@ -4,12 +4,8 @@ import java.util.Collection;
 import java.util.HashMap;
 import java.util.List;
 import java.util.Map;
+import java.util.Properties;
 
-import org.apache.log4j.Logger;
-
-import ch.systemsx.cisd.common.logging.LogCategory;
-import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.common.properties.ExtendedProperties;
 import ch.systemsx.cisd.common.properties.PropertyUtils;
 import ch.systemsx.cisd.etlserver.IAutoArchiverPolicy;
 import ch.systemsx.cisd.etlserver.plugins.grouping.DatasetListWithTotal;
@@ -26,9 +22,6 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
  */
 public abstract class BaseGroupingPolicy implements IAutoArchiverPolicy
 {
-    private static final Logger operationLog =
-            LogFactory.getLogger(LogCategory.OPERATION, BaseGroupingPolicy.class);
-
     public static final String MINIMAL_ARCHIVE_SIZE = "minimal-archive-size";
 
     public static final String MAXIMAL_ARCHIVE_SIZE = "maximal-archive-size";
@@ -43,7 +36,7 @@ public abstract class BaseGroupingPolicy implements IAutoArchiverPolicy
 
     private IDataSetPathInfoProvider pathInfoProvider;
 
-    public BaseGroupingPolicy(ExtendedProperties properties)
+    public BaseGroupingPolicy(Properties properties)
     {
         minArchiveSize =
                 PropertyUtils.getLong(properties, MINIMAL_ARCHIVE_SIZE, DEFAULT_MINIMAL_ARCHIVE_SIZE);
@@ -55,6 +48,10 @@ public abstract class BaseGroupingPolicy implements IAutoArchiverPolicy
     @Override
     public final List<AbstractExternalData> filter(List<AbstractExternalData> dataSets)
     {
+        if (dataSets.isEmpty())
+        {
+            return dataSets;
+        }
         makeSureAllDataSetsWithSize(dataSets);
         return filterDataSetsWithSizes(dataSets);
     }

datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ByExperimentPolicy.java

@@ -4,13 +4,9 @@ import java.util.ArrayList;
 import java.util.Collection;
 import java.util.Collections;
 import java.util.List;
-
-import org.apache.log4j.Logger;
+import java.util.Properties;
 
 import ch.systemsx.cisd.common.collection.SimpleComparator;
-import ch.systemsx.cisd.common.logging.LogCategory;
-import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.common.properties.ExtendedProperties;
 import ch.systemsx.cisd.etlserver.IAutoArchiverPolicy;
 import ch.systemsx.cisd.etlserver.plugins.grouping.DatasetListWithTotal;
 import ch.systemsx.cisd.etlserver.plugins.grouping.Grouping;
@@ -25,12 +21,9 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
  */
 public class ByExperimentPolicy extends BaseGroupingPolicy implements IAutoArchiverPolicy
 {
-    private static final Logger operationLog =
-            LogFactory.getLogger(LogCategory.OPERATION, ByExperimentPolicy.class);
-
     private final List<IGroupKeyProvider> providers;
 
-    public ByExperimentPolicy(ExtendedProperties properties)
+    public ByExperimentPolicy(Properties properties)
     {
         super(properties);
         providers = new ArrayList<IGroupKeyProvider>();

datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BySpacePolicy.java

@@ -5,12 +5,8 @@ import java.util.Collection;
 import java.util.Collections;
 import java.util.LinkedList;
 import java.util.List;
+import java.util.Properties;
 
-import org.apache.log4j.Logger;
-
-import ch.systemsx.cisd.common.logging.LogCategory;
-import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.common.properties.ExtendedProperties;
 import ch.systemsx.cisd.etlserver.IAutoArchiverPolicy;
 import ch.systemsx.cisd.etlserver.plugins.grouping.DatasetListWithTotal;
 import ch.systemsx.cisd.etlserver.plugins.grouping.Grouping;
@@ -24,12 +20,9 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
  */
 public class BySpacePolicy extends BaseGroupingPolicy implements IAutoArchiverPolicy
 {
-    private static final Logger operationLog =
-            LogFactory.getLogger(LogCategory.OPERATION, BySpacePolicy.class);
-
     private final List<IGroupKeyProvider> providers;
 
-    public BySpacePolicy(ExtendedProperties properties)
+    public BySpacePolicy(Properties properties)
     {
         super(properties);
         providers = new ArrayList<IGroupKeyProvider>();

datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/GroupingPolicy.java

+/*
+ * Copyright 2015 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.text.SimpleDateFormat;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.collection.CollectionUtils;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.properties.PropertyUtils;
+import ch.systemsx.cisd.etlserver.plugins.grouping.DatasetListWithTotal;
+import ch.systemsx.cisd.etlserver.plugins.grouping.Grouping;
+import ch.systemsx.cisd.etlserver.plugins.grouping.IGroupKeyProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.BasicConstant;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Code;
+
+/**
+ * Configurable auto archiving policy which allows to find a group of data sets with total size
+ * from a specified interval. Grouping can be defined by space, project, experiment, sample, data set type or
+ * a combination of those. The combination defines a so called 'grouping key'. 
+ * Groups can be merged if they are too small. Several grouping keys can be specified.
+ * <p>
+ * Searching for an appropriate group of data sets for auto archiving is logged. If no group could be found
+ * the admin (as specified in log.xml) is notified be email with the searching log as content. 
+ *
+ * @author Franz-Josef Elmer
+ */
+public class GroupingPolicy extends BaseGroupingPolicy
+{
+    static final String GROUPING_KEYS_KEY = "grouping-keys";
+    private static final Logger operationLog =
+            LogFactory.getLogger(LogCategory.OPERATION, GroupingPolicy.class);
+    private static final Logger notificationLog =
+            LogFactory.getLogger(LogCategory.NOTIFY, GroupingPolicy.class);
+   
+    private final List<CombinedGroupKeys> groupKeyProviders = new ArrayList<CombinedGroupKeys>();
+
+    public GroupingPolicy(Properties properties)
+    {
+        super(properties);
+        List<String> groupingKeys = PropertyUtils.getList(properties, GROUPING_KEYS_KEY);
+        for (String groupingKey : groupingKeys)
+        {
+            String[] splitted = groupingKey.split(":", 2);
+            boolean merge = false;
+            if (splitted.length > 1)
+            {
+                if ("merge".equals(splitted[1]) == false)
+                {
+                    throw new ConfigurationFailureException("Invalid grouping key in property '" + GROUPING_KEYS_KEY 
+                            + "' because 'merge' is expected after ':': " + groupingKey);
+                }
+                merge = splitted.length < 2 ? false : "merge".equals(splitted[1]);
+            }
+            String[] keyItems = splitted[0].split("#");
+            List<IGroupKeyProvider> groupings = new ArrayList<IGroupKeyProvider>();
+            for (String keyItem : keyItems)
+            {
+                try
+                {
+                    groupings.add(Grouping.valueOf(keyItem));
+                } catch (IllegalArgumentException ex)
+                {
+                    throw new ConfigurationFailureException("Invalid basic grouping key in property '" 
+                            + GROUPING_KEYS_KEY + "': " + keyItem + " (valid values are " 
+                            + Arrays.asList(Grouping.values()) + ")");
+                }
+            }
+            groupKeyProviders.add(new CombinedGroupKeys(groupings, merge));
+        }
+    }
+
+    @Override
+    protected List<AbstractExternalData> filterDataSetsWithSizes(List<AbstractExternalData> dataSets)
+    {
+        List<String> log = new ArrayList<String>();
+        for (CombinedGroupKeys combinedGroupKeys : groupKeyProviders)
+        {
+            List<DatasetListWithTotal> groups = splitIntoGroups(dataSets, combinedGroupKeys);
+            log(log, combinedGroupKeys + " has grouped " + dataSets.size() + " data sets into " 
+                    + groups.size() + " groups.");
+            if (groups.isEmpty() == false)
+            {
+                List<AbstractExternalData> result 
+                    = tryFindGroupOrMerge(groups, combinedGroupKeys.isMerge(), log);
+                if (result != null)
+                {
+                    log(log, "filtered data sets: " + CollectionUtils.abbreviate(Code.extractCodes(result), 20));
+                    return result;
+                }
+            }
+        }
+        StringBuilder builder = new StringBuilder();
+        builder.append("From " + dataSets.size() + " data sets no group could be found to be fit between ");
+        builder.append(FileUtils.byteCountToDisplaySize(minArchiveSize)).append(" and ");
+        builder.append(FileUtils.byteCountToDisplaySize(maxArchiveSize));
+        builder.append("\n\nLog:");
+        for (String logMessage : log)
+        {
+            builder.append('\n').append(logMessage);
+        }
+        notificationLog.warn(builder.toString());
+        return new ArrayList<AbstractExternalData>();
+    }
+    
+    private List<AbstractExternalData> tryFindGroupOrMerge(List<DatasetListWithTotal> groups, 
+            boolean merge, List<String> log)
+    {
+        List<DatasetListWithTotal> tooSmallGroups = new ArrayList<DatasetListWithTotal>();
+        List<DatasetListWithTotal> fittingGroups = new ArrayList<DatasetListWithTotal>();
+        for (DatasetListWithTotal group : groups)
+        {
+            long size = group.getCumulatedSize();
+            if (size < minArchiveSize)
+            {
+                tooSmallGroups.add(group);
+            } else if (size <= maxArchiveSize)
+            {
+                fittingGroups.add(group);
+            }
+        }
+        log(log, fittingGroups.size() + " groups match in size, " + tooSmallGroups.size() + " groups are too small and " 
+                + (groups.size() - fittingGroups.size() - tooSmallGroups.size()) + " groups are too large.");
+        if (fittingGroups.isEmpty() == false)
+        {
+            return getOldestGroup(fittingGroups, log);
+        }
+        if (tooSmallGroups.size() < 2 || merge == false)
+        {
+            return null;
+        }
+        return tryMerge(tooSmallGroups, log);
+    }
+
+    private List<AbstractExternalData> getOldestGroup(List<DatasetListWithTotal> groups, List<String> log)
+    {
+        if (groups.size() == 1)
+        {
+            return groups.get(0).getList();
+        }
+        GroupWithAge oldestGroup = sortGroupsByAge(groups).get(0);
+        String timestamp = new SimpleDateFormat(BasicConstant.DATE_WITHOUT_TIMEZONE_PATTERN).format(new Date(oldestGroup.age));
+        log(log, "All data sets have been accessed at " + timestamp + " or before.");
+        return oldestGroup.group.getList();
+    }
+
+    private List<GroupWithAge> sortGroupsByAge(List<DatasetListWithTotal> groups)
+    {
+        List<GroupWithAge> groupsWithAge = new ArrayList<GroupWithAge>();
+        for (DatasetListWithTotal group : groups)
+        {
+            groupsWithAge.add(new GroupWithAge(group));
+        }
+        Collections.sort(groupsWithAge);
+        return groupsWithAge;
+    }
+    
+    private List<AbstractExternalData> tryMerge(List<DatasetListWithTotal> groups, List<String> log)
+    {
+        List<GroupWithAge> groupsWithAge = sortGroupsByAge(groups);
+        List<AbstractExternalData> result = new ArrayList<AbstractExternalData>();
+        long total = 0;
+        for (int i = 0; i < groupsWithAge.size(); i++)
+        {
+            DatasetListWithTotal group = groupsWithAge.get(i).group;
+            result.addAll(group.getList());
+            total += group.getCumulatedSize();
+            if (total >= minArchiveSize)
+            {
+                if (total <= maxArchiveSize)
+                {
+                    log(log, (i+1) + " groups have been merged.");
+                    return result;
+                }
+                log(log, (i+1) + " groups have been merged, but the total size of " + FileUtils.byteCountToDisplaySize(total) 
+                        + " is above the required maximum of " + FileUtils.byteCountToDisplaySize(maxArchiveSize));
+                return null;
+            }
+        }
+        log(log, "Merging all " + groups.size() + " groups gives a total size of " + FileUtils.byteCountToDisplaySize(total) 
+                + " which is still below required minimum of " + FileUtils.byteCountToDisplaySize(minArchiveSize));
+        return null;
+    }
+    
+    private List<DatasetListWithTotal> splitIntoGroups(List<AbstractExternalData> dataSets, IGroupKeyProvider groupKeyProvider)
+    {
+        List<DatasetListWithTotal> groups = new ArrayList<DatasetListWithTotal>(
+                splitDataSetsInGroupsAccordingToCriteria(dataSets, groupKeyProvider));
+        Collections.sort(groups);
+        return groups;
+    }
+    
+    private void log(List<String> log, Object logMessage)
+    {
+        log.add(logMessage.toString());
+        operationLog.info(logMessage.toString());
+    }
+
+    private static final class CombinedGroupKeys implements IGroupKeyProvider
+    {
+        private final List<IGroupKeyProvider> groupKeyProviders;
+        private final boolean merge;
+
+        CombinedGroupKeys(List<IGroupKeyProvider> groupKeyProviders, boolean merge)
+        {
+            this.groupKeyProviders = groupKeyProviders;
+            this.merge = merge;
+        }
+
+        public boolean isMerge()
+        {
+            return merge;
+        }
+
+        @Override
+        public String getGroupKey(AbstractExternalData dataset)
+        {
+            StringBuilder builder = new StringBuilder();
+            for (IGroupKeyProvider groupKeyProvider : groupKeyProviders)
+            {
+                if (builder.length() > 0)
+                {
+                    builder.append('#');
+                }
+                builder.append(groupKeyProvider.getGroupKey(dataset));
+            }
+            return builder.toString();
+        }
+
+        @Override
+        public String toString()
+        {
+            StringBuilder builder = new StringBuilder();
+            for (IGroupKeyProvider keyProvider : groupKeyProviders)
+            {
+                if (builder.length() > 0)
+                {
+                    builder.append('#');
+                }
+                builder.append(keyProvider);
+            }
+            if (merge)
+            {
+                builder.append(":merge");
+            }
+            return "Grouping key: '" + builder.toString() + "'";
+        }
+        
+    }
+
+    private static class GroupWithAge implements Comparable<GroupWithAge>
+    {
+        private long age;
+        private DatasetListWithTotal group;
+        GroupWithAge(DatasetListWithTotal group)
+        {
+            this.group = group;
+            List<AbstractExternalData> dataSets = group.getList();
+            for (AbstractExternalData dataSet : dataSets)
+            {
+                age = Math.max(age, dataSet.getAccessTimestamp().getTime());
+            }
+        }
+        @Override
+        public int compareTo(GroupWithAge that)
+        {
+            return Long.signum(this.age - that.age);
+        }
+    }
+}

datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/grouping/Grouping.java

@@ -22,9 +22,16 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
 /**
  * @author Jakub Straszewski
  */
-
 public enum Grouping implements IGroupKeyProvider
 {
+    All
+    {
+        @Override
+        public String getGroupKey(AbstractExternalData dataset)
+        {
+            return "all";
+        }
+    },
     Space
     {
         @Override

datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/AbstractAutoArchiverPolicyTestCase.java

+/*
+ * Copyright 2015 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Date;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.testng.AssertJUnit;
+import org.testng.annotations.BeforeMethod;
+
+import ch.systemsx.cisd.common.properties.ExtendedProperties;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Code;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PhysicalDataSet;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space;
+
+/**
+ * 
+ *
+ * @author Franz-Josef Elmer
+ */
+public class AbstractAutoArchiverPolicyTestCase extends AssertJUnit
+{
+
+
+    protected static final class ExecutionContext
+    {
+        private AtomicInteger counter = new AtomicInteger();
+
+        public AbstractExternalData createDataset(String projectCode, String experimentCode, 
+                String datasetType, String dsCode, Long size)
+        {
+            return createDataset("___space", projectCode, experimentCode, datasetType, dsCode, size);
+        }
+
+        /**
+         * If datasetCode is null then it gets assigned unique code containing the word "generated"
+         */
+        public AbstractExternalData createDataset(String spaceCode, String projectCode, String experimentCode, 
+                String datasetType, String dsCode, Long accessTimestamp, Long size)
+        {
+            return createDataset(spaceCode, projectCode, experimentCode, datasetType, null, dsCode, accessTimestamp, size);
+        }
+
+        /**
+         * If datasetCode is null then it gets assigned unique code containing the word "generated"
+         */
+        public AbstractExternalData createDataset(String spaceCode, String projectCode, String experimentCode, 
+                String datasetType, String dsCode, Long size)
+        {
+            return createDataset(spaceCode, projectCode, experimentCode, datasetType, null, dsCode, size);
+        }
+        
+        /**
+         * If datasetCode is null then it gets assigned unique code containing the word "generated"
+         */
+        public AbstractExternalData createDataset(String spaceCode, String projectCode, String experimentCode, 
+                String datasetType, String sampleCode, String dsCode, Long size)
+        {
+            return createDataset(spaceCode, projectCode, experimentCode, datasetType, sampleCode, dsCode, null, size);
+        }
+        
+        /**
+         * If datasetCode is null then it gets assigned unique code containing the word "generated"
+         */
+        public AbstractExternalData createDataset(String spaceCode, String projectCode, String experimentCode, 
+                String datasetType, String sampleCode, String dsCode, Long accessTimestamp, Long size)
+        {
+            Space space = new Space();
+            space.setCode(spaceCode);
+            space.setIdentifier("/" + space.getCode());
+
+            Project project = new Project();
+            project.setCode(projectCode);
+            project.setIdentifier("/" + spaceCode + "/" + projectCode);
+            project.setSpace(space);
+
+            Experiment exp = new Experiment();
+            exp.setProject(project);
+            exp.setCode(experimentCode);
+            exp.setIdentifier(project.getIdentifier() + "/" + experimentCode);
+
+            Sample sample = null;
+            if (sampleCode != null)
+            {
+                sample = new Sample();
+                sample.setCode(sampleCode);
+                sample.setIdentifier(space.getIdentifier() + "/" + sample.getCode());
+                sample.setExperiment(exp);
+                sample.setSpace(space);
+            }
+
+            DataSetType dataSetType = new DataSetType();
+            dataSetType.setCode(datasetType);
+
+            PhysicalDataSet ds = new PhysicalDataSet();
+            if (dsCode != null)
+            {
+                ds.setCode(dsCode);
+            }
+            else
+            {
+                ds.setCode("generated-" + counter.incrementAndGet());
+            }
+            ds.setExperiment(exp);
+            ds.setSample(sample);
+            ds.setSize(size);
+            ds.setDataSetType(dataSetType);
+            ds.setAccessTimestamp(accessTimestamp == null ? new Date(0) : new Date(accessTimestamp));
+
+            return ds;
+        }
+    }
+
+    protected ExecutionContext ctx;
+
+    @BeforeMethod
+    public void setUp()
+    {
+        ctx = new ExecutionContext();
+    }
+    
+    protected ExtendedProperties createPolicyProperties(long min, long max)
+    {
+        ExtendedProperties props = new ExtendedProperties();
+        props.setProperty(BaseGroupingPolicy.MINIMAL_ARCHIVE_SIZE, Long.toString(min));
+        props.setProperty(BaseGroupingPolicy.MAXIMAL_ARCHIVE_SIZE, Long.toString(max));
+        return props;
+    }
+
+}

datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/ByExperimentPolicyTest.java

@@ -9,6 +9,7 @@ import ch.systemsx.cisd.common.properties.ExtendedProperties;
 import ch.systemsx.cisd.common.test.AssertionUtil;
 import ch.systemsx.cisd.etlserver.IAutoArchiverPolicy;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Code;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
 
 /**
@@ -37,7 +38,7 @@ public class ByExperimentPolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(14, 100, dataSets);
 
-        assertEquals("[ds1, ds2, ds3]", extractCodes(filtered).toString());
+        assertEquals("[ds1, ds2, ds3]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -52,7 +53,7 @@ public class ByExperimentPolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(15, 25, dataSets);
 
-        assertEquals("[ds2, ds3]", extractCodes(filtered).toString());
+        assertEquals("[ds2, ds3]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -68,7 +69,7 @@ public class ByExperimentPolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filterWithoutShuffling(6, 10, dataSets);
 
-        assertEquals("[ds1]", extractCodes(filtered).toString());
+        assertEquals("[ds1]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -84,7 +85,7 @@ public class ByExperimentPolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(500, 1000, dataSets);
 
-        assertEquals("[]", extractCodes(filtered).toString());
+        assertEquals("[]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -117,7 +118,7 @@ public class ByExperimentPolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(10, 1000, dataSets);
 
-        assertEquals("[ds1, ds2]", extractCodes(filtered).toString());
+        assertEquals("[ds1, ds2]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -135,7 +136,7 @@ public class ByExperimentPolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(10, 1000, dataSets);
 
-        assertEquals("[ds1, ds2]", extractCodes(filtered).toString());
+        assertEquals("[ds1, ds2]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -152,7 +153,7 @@ public class ByExperimentPolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(10, 1000, dataSets);
 
-        assertEquals("[ds1, ds2, ds3]", extractCodes(filtered).toString());
+        assertEquals("[ds1, ds2, ds3]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -170,7 +171,7 @@ public class ByExperimentPolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(10, 1000, dataSets);
 
-        assertEquals("[ds3]", extractCodes(filtered).toString());
+        assertEquals("[ds3]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }

datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/ByPoliceAbstractTest.java

@@ -20,13 +20,11 @@ import java.util.ArrayList;
 import java.util.Collections;
 import java.util.List;
 import java.util.Random;
-import java.util.concurrent.atomic.AtomicInteger;
 
 import org.jmock.Expectations;
 import org.jmock.Mockery;
 import org.springframework.beans.factory.BeanFactory;
 import org.springframework.test.annotation.ExpectedException;
-import org.testng.AssertJUnit;
 import org.testng.ITestResult;
 import org.testng.annotations.AfterMethod;
 import org.testng.annotations.BeforeMethod;
@@ -40,34 +38,23 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.ISingleDataSetPathInfoProvide
 import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProviderTestWrapper;
 import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetPathInfo;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PhysicalDataSet;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Space;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Code;
 
 /**
  * @author Jakub Straszewski
  */
-public abstract class ByPoliceAbstractTest extends AssertJUnit
+public abstract class ByPoliceAbstractTest extends AbstractAutoArchiverPolicyTestCase
 {
     protected Mockery context;
 
-    protected ExecutionContext ctx;
-
     private IDataSetPathInfoProvider pathProviderMock;
 
     private ISingleDataSetPathInfoProvider singleDsProviderMock;
 
-    private AtomicInteger counter;
-
     @BeforeMethod
     public void setUpTestEnvironment()
     {
-        ctx = new ExecutionContext();
         context = new Mockery();
-        counter = new AtomicInteger();
         final BeanFactory beanFactory = context.mock(BeanFactory.class);
         ServiceProviderTestWrapper.setApplicationContext(beanFactory);
         pathProviderMock = ServiceProviderTestWrapper.mock(context, IDataSetPathInfoProvider.class);
@@ -114,84 +101,6 @@ public abstract class ByPoliceAbstractTest extends AssertJUnit
         ServiceProviderTestWrapper.restoreApplicationContext();
     }
 
-    protected class ExecutionContext
-    {
-        public AbstractExternalData createDataset(String projectCode, String experimentCode, String datasetType, String dsCode, Long size)
-        {
-            return createDataset("___space", projectCode, experimentCode, datasetType, dsCode, size);
-        }
-
-        /**
-         * If datasetCode is null then it gets assigned unique code containing the word "generated"
-         */
-        public AbstractExternalData createDataset(String spaceCode, String projectCode, String experimentCode, String datasetType, String dsCode,
-                Long size)
-        {
-            return createDataset(spaceCode, projectCode, experimentCode, datasetType, null, dsCode, size);
-        }
-
-        /**
-         * If datasetCode is null then it gets assigned unique code containing the word "generated"
-         */
-        public AbstractExternalData createDataset(String spaceCode, String projectCode, String experimentCode, String datasetType, String sampleCode,
-                String dsCode,
-                Long size)
-        {
-            Space space = new Space();
-            space.setCode(spaceCode);
-            space.setIdentifier("/" + space.getCode());
-
-            Project project = new Project();
-            project.setCode(projectCode);
-            project.setIdentifier("/" + spaceCode + "/" + projectCode);
-            project.setSpace(space);
-
-            Experiment exp = new Experiment();
-            exp.setProject(project);
-            exp.setCode(experimentCode);
-            exp.setIdentifier(project.getIdentifier() + "/" + experimentCode);
-
-            Sample sample = null;
-            if (sampleCode != null)
-            {
-                sample = new Sample();
-                sample.setCode(sampleCode);
-                sample.setIdentifier(space.getIdentifier() + "/" + sample.getCode());
-                sample.setExperiment(exp);
-                sample.setSpace(space);
-            }
-
-            DataSetType dataSetType = new DataSetType();
-            dataSetType.setCode(datasetType);
-
-            PhysicalDataSet ds = new PhysicalDataSet();
-            if (dsCode != null)
-            {
-                ds.setCode(dsCode);
-            }
-            else
-            {
-                ds.setCode("generated-" + counter.incrementAndGet());
-            }
-            ds.setExperiment(exp);
-            ds.setSample(sample);
-            ds.setSize(size);
-            ds.setDataSetType(dataSetType);
-
-            return ds;
-        }
-    }
-
-    protected List<String> extractCodes(List<AbstractExternalData> dataSets)
-    {
-        List<String> codes = new ArrayList<String>();
-        for (AbstractExternalData dataSet : dataSets)
-        {
-            codes.add(dataSet.getCode());
-        }
-        Collections.sort(codes);
-        return codes;
-    }
 
     // Some general tests for all policies
 
@@ -201,7 +110,7 @@ public abstract class ByPoliceAbstractTest extends AssertJUnit
      * Creates the policy with given min and max value. To improve the robusntess of the test this method shuffles the incoming dataset and sorts the
      * result.
      */
-    protected List<AbstractExternalData> filter(int min, int max, ArrayList<AbstractExternalData> dataSets)
+    protected List<AbstractExternalData> filter(int min, int max, List<AbstractExternalData> dataSets)
     {
         return filter(min, max, dataSets, true);
     }
@@ -209,12 +118,12 @@ public abstract class ByPoliceAbstractTest extends AssertJUnit
     /**
      * Creates the policy with given min and max value. The input to the filtering is not shuffled.
      */
-    protected List<AbstractExternalData> filterWithoutShuffling(int min, int max, ArrayList<AbstractExternalData> dataSets)
+    protected List<AbstractExternalData> filterWithoutShuffling(int min, int max, List<AbstractExternalData> dataSets)
     {
         return filter(min, max, dataSets, false);
     }
 
-    private List<AbstractExternalData> filter(int min, int max, ArrayList<AbstractExternalData> dataSets, boolean shuffle)
+    private List<AbstractExternalData> filter(int min, int max, List<AbstractExternalData> dataSets, boolean shuffle)
     {
         IAutoArchiverPolicy policy = getPolicy(min, max);
 
@@ -292,7 +201,7 @@ public abstract class ByPoliceAbstractTest extends AssertJUnit
 
     protected void assertAllDataSetsAreNotGenerated(List<AbstractExternalData> filtered)
     {
-        if (extractCodes(filtered).toString().contains("generated"))
+        if (Code.extractCodes(filtered).toString().contains("generated"))
         {
             fail(getErrorMessage(filtered));
         }
@@ -300,7 +209,7 @@ public abstract class ByPoliceAbstractTest extends AssertJUnit
 
     protected String getErrorMessage(List<AbstractExternalData> filtered)
     {
-        return "Unexpected data sets in result of filtering data sets." + extractCodes(filtered).toString();
+        return "Unexpected data sets in result of filtering data sets." + Code.extractCodes(filtered).toString();
     }
 
     protected void assertTotalDataSetsSize(long expectedSize, List<AbstractExternalData> dataSets)

datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/BySpacePolicyTest.java

@@ -11,6 +11,7 @@ import ch.systemsx.cisd.common.properties.ExtendedProperties;
 import ch.systemsx.cisd.common.test.AssertionUtil;
 import ch.systemsx.cisd.etlserver.plugins.grouping.Grouping;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Code;
 
 /**
  * @author Sascha Fedorenko
@@ -60,7 +61,7 @@ public class BySpacePolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(14, 100, dataSets);
 
-        assertEquals("[ds1, ds2, ds3]", extractCodes(filtered).toString());
+        assertEquals("[ds1, ds2, ds3]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -76,7 +77,7 @@ public class BySpacePolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(50, 100, dataSets);
 
-        assertEquals("[]", extractCodes(filtered).toString());
+        assertEquals("[]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -93,7 +94,7 @@ public class BySpacePolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(40, 100, dataSets);
 
-        assertEquals("[ds1, ds2, ds3, ds4]", extractCodes(filtered).toString());
+        assertEquals("[ds1, ds2, ds3, ds4]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -115,7 +116,7 @@ public class BySpacePolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(2, 4, dataSets);
 
-        assertEquals("[ds6, ds7, ds8]", extractCodes(filtered).toString());
+        assertEquals("[ds6, ds7, ds8]", Code.extractCodes(filtered).toString());
 
         context.assertIsSatisfied();
     }
@@ -190,7 +191,7 @@ public class BySpacePolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(5, 8, dataSets);
 
-        assertEquals(extractCodes(filtered).toString(), "[a, b, c, d, e, f]");
+        assertEquals(Code.extractCodes(filtered).toString(), "[a, b, c, d, e, f]");
 
         context.assertIsSatisfied();
     }
@@ -276,7 +277,7 @@ public class BySpacePolicyTest extends ByPoliceAbstractTest
 
         List<AbstractExternalData> filtered = filter(100, 140, dataSets);
 
-        assertEquals(extractCodes(filtered).toString(), "[a, b, c]");
+        assertEquals(Code.extractCodes(filtered).toString(), "[a, b, c]");
 
         context.assertIsSatisfied();
     }
@@ -354,7 +355,7 @@ public class BySpacePolicyTest extends ByPoliceAbstractTest
         dataSets.add(ctx.createDataset("rightSpace", "rightProject", "smallE2", "dt1", null, 60L));
         dataSets.add(ctx.createDataset("rightSpace", "rightProject", "smallE3", "dt1", null, 30L));
         dataSets.add(ctx.createDataset("rightSpace", "rightProject", "smallE3", "dt1", null, 30L));
-        dataSets.add(ctx.createDataset("rightSpace", "rightProject", "bigE", "dt1", "toSmallSample", null, 20L));
+        dataSets.add(ctx.createDataset("rightSpace", "rightProject", "bigE", "dt1", "toSmallSample", (String) null, 20L));
         dataSets.add(ctx.createDataset("rightSpace", "rightProject", "bigE", "dt1", null, "a", 20L));
         dataSets.add(ctx.createDataset("rightSpace", "rightProject", "bigE", "dt1", null, "b", 20L));
         dataSets.add(ctx.createDataset("rightSpace", "rightProject", "bigE", "dt1", null, "c", 20L));