From 9f57793119c0e3a566bd5c07cd74cc4df8cccea1 Mon Sep 17 00:00:00 2001
From: izabel <izabel>
Date: Tue, 1 Sep 2009 23:27:43 +0000
Subject: [PATCH] [SE-131] metabol db updating plugin
SVN: 12354
---
.../ch/systemsx/cisd/etlserver/ETLDaemon.java | 17 ++
.../cisd/etlserver/IMaintenanceTask.java | 37 ++++
.../cisd/etlserver/MaintenancePlugin.java | 43 ++++
.../etlserver/MaintenanceTaskParameters.java | 61 ++++++
.../systemsx/cisd/etlserver/Parameters.java | 38 ++++
.../cisd/etlserver/plugins/.gitignore | 0
.../plugins/HierarchicalStorageUpdater.java | 201 ++++++++++++++++++
.../openbis/dss/generic/DataStoreServer.java | 1 -
.../server/DataSetHierarchyHelper.java | 2 +-
.../server/HierarchicalStorageDaemon.java | 1 +
.../dss/generic/server/SoftLinkMaker.java | 2 +-
.../openbis/generic/server/ETLService.java | 2 +-
.../server/dataaccess/db/EventDAO.java | 4 +-
.../server/dataaccess/db/EventDAOTest.java | 9 +
.../yeastx/etl/MetabolDatabaseUpdater.java | 162 ++++++++++++++
.../source/sql/postgresql/002/schema-002.sql | 19 +-
.../migration/migration-001-002.sql | 12 ++
17 files changed, 591 insertions(+), 20 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/IMaintenanceTask.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/MaintenancePlugin.java
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/MaintenanceTaskParameters.java
delete mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/.gitignore
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
create mode 100644 rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MetabolDatabaseUpdater.java
create mode 100644 rtd_yeastx/source/sql/postgresql/migration/migration-001-002.sql
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
index 0fd75967808..dcafaf3d3b8 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/ETLDaemon.java
@@ -546,7 +546,24 @@ public final class ETLDaemon
QueueingPathRemoverService.start(shredderQueueFile);
printInitialLogMessage(parameters);
startupServer(parameters);
+ startupMaintenancePlugins(parameters.getMaintenancePlugins());
operationLog.info("Data Store Server ready and waiting for data.");
}
+ private static void startupMaintenancePlugins(MaintenanceTaskParameters[] maintenancePlugins)
+ {
+
+ List<MaintenancePlugin> plugins = new ArrayList<MaintenancePlugin>();
+ for (MaintenanceTaskParameters parameters : maintenancePlugins)
+ {
+ MaintenancePlugin plugin = new MaintenancePlugin(parameters);
+ plugins.add(plugin);
+ }
+ for (MaintenancePlugin plugin : plugins)
+ {
+ plugin.start();
+ }
+
+ }
+
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IMaintenanceTask.java
new file mode 100644
index 00000000000..b2ca39f8079
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IMaintenanceTask.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+/**
+ * The interface that should be implemented by all maintenance tasks.
+ *
+ * @author Izabela Adamczyk
+ */
+public interface IMaintenanceTask
+{
+
+ /**
+ * Performs the maintenance task.
+ */
+ public void execute();
+
+ /**
+ * Prepares the task for execution and checks that it has been configured correctly.
+ */
+ public void setUp(String pluginName);
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/MaintenancePlugin.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/MaintenancePlugin.java
new file mode 100644
index 00000000000..f59524e5c4f
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/MaintenancePlugin.java
@@ -0,0 +1,43 @@
+package ch.systemsx.cisd.etlserver;
+
+import java.util.Timer;
+import java.util.TimerTask;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.utilities.ClassUtils;
+
+public class MaintenancePlugin
+{
+ IMaintenanceTask task;
+
+ MaintenanceTaskParameters parameters;
+
+ public MaintenancePlugin(MaintenanceTaskParameters parameters)
+ {
+ this.parameters = parameters;
+ try
+ {
+ this.task = ClassUtils.create(IMaintenanceTask.class, parameters.getClassName());
+ } catch (Exception ex)
+ {
+ throw new ConfigurationFailureException("Cannot find the plugin class '" + parameters
+ + "'", CheckedExceptionTunnel.unwrapIfNecessary(ex));
+ }
+ task.setUp(parameters.getPluginName());
+ }
+
+ public void start()
+ {
+ final String timerThreadName = parameters.getPluginName() + " - Maintenance Plugin";
+ final Timer workerTimer = new Timer(timerThreadName);
+ workerTimer.schedule(new TimerTask()
+ {
+ @Override
+ public void run()
+ {
+ task.execute();
+ }
+ }, 0L, parameters.getInterval() * 1000);
+ }
+}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/MaintenanceTaskParameters.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/MaintenanceTaskParameters.java
new file mode 100644
index 00000000000..76262bbf9c7
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/MaintenanceTaskParameters.java
@@ -0,0 +1,61 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver;
+
+import java.util.Properties;
+
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+
+/**
+ * @author Izabela Adamczyk
+ */
+public class MaintenanceTaskParameters
+{
+ private static final int ONE_DAY_IN_SEC = 60 * 60 * 24;
+
+ private static final String CLASS_KEY = "class";
+
+ private static final String INTERVAL_KEY = "interval";
+
+ private final String pluginName;
+
+ private final long interval;
+
+ private final String className;
+
+ public MaintenanceTaskParameters(Properties properties, String pluginName)
+ {
+ this.pluginName = pluginName;
+ interval = PropertyUtils.getLong(properties, INTERVAL_KEY, ONE_DAY_IN_SEC);
+ className = PropertyUtils.getProperty(properties, CLASS_KEY);
+ }
+
+ public long getInterval()
+ {
+ return interval;
+ }
+
+ public String getClassName()
+ {
+ return className;
+ }
+
+ public String getPluginName()
+ {
+ return pluginName;
+ }
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/Parameters.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/Parameters.java
index 76eae9367a0..6d94aa059e4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/Parameters.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/Parameters.java
@@ -63,6 +63,11 @@ public class Parameters
/** property with thread names separated by delimiter */
private static final String INPUT_THREAD_NAMES = "inputs";
+ /**
+ * property with maintenance plugin names separated by delimiter
+ */
+ private static final String MAINTENANCE_PLUGINS = "maintenance-plugins";
+
@Option(name = "s", longName = "server-url", metaVar = "URL", usage = "URL of the server")
private String serverURL;
@@ -129,6 +134,8 @@ public class Parameters
private final Map<String, Properties> processorProperties;
+ private MaintenanceTaskParameters[] maintenancePlugins;
+
@Option(longName = "help", skipForExample = true, usage = "Prints out a description of the options.")
void printHelp(final boolean exit)
{
@@ -177,6 +184,7 @@ public class Parameters
this.threads = createThreadParameters(serviceProperties);
this.mailProperties = createMailProperties(serviceProperties);
this.timingParameters = TimingParameters.create(serviceProperties);
+ this.maintenancePlugins = createMaintenancePlugins(serviceProperties);
initCommandLineParametersFromProperties();
@@ -191,6 +199,11 @@ public class Parameters
}
}
+ public MaintenanceTaskParameters[] getMaintenancePlugins()
+ {
+ return maintenancePlugins;
+ }
+
private void ensureParametersCorrect()
{
for (final ThreadParameters thread : threads)
@@ -252,6 +265,31 @@ public class Parameters
}
}
+ private static MaintenanceTaskParameters[] createMaintenancePlugins(
+ final Properties serviceProperties)
+ {
+ SectionProperties[] sectionsProperties =
+ PropertyParametersUtil.extractSectionProperties(serviceProperties,
+ MAINTENANCE_PLUGINS, true);
+ return asMaintenanceParameters(sectionsProperties);
+ }
+
+ private static MaintenanceTaskParameters[] asMaintenanceParameters(
+ SectionProperties[] sectionProperties)
+ {
+ final MaintenanceTaskParameters[] maintenanceParameters =
+ new MaintenanceTaskParameters[sectionProperties.length];
+ for (int i = 0; i < maintenanceParameters.length; i++)
+ {
+ SectionProperties section = sectionProperties[i];
+ operationLog.info("Create parameters for maintenance plugin '" + section.getKey()
+ + "'.");
+ maintenanceParameters[i] =
+ new MaintenanceTaskParameters(section.getProperties(), section.getKey());
+ }
+ return maintenanceParameters;
+ }
+
private static ThreadParameters[] asThreadParameters(SectionProperties[] sectionProperties)
{
final ThreadParameters[] threadParameters = new ThreadParameters[sectionProperties.length];
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/.gitignore b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/.gitignore
deleted file mode 100644
index e69de29bb2d..00000000000
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
new file mode 100644
index 00000000000..ad4479fd864
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/HierarchicalStorageUpdater.java
@@ -0,0 +1,201 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import java.io.File;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Properties;
+import java.util.Set;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.process.ProcessExecutionHelper;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.etlserver.IMaintenanceTask;
+import ch.systemsx.cisd.openbis.dss.generic.server.DataSetHierarchyHelper;
+import ch.systemsx.cisd.openbis.dss.generic.server.SoftLinkMaker;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.utils.PropertyParametersUtil;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
+
+/**
+ * Creates the hierarchical structure of data sets registered in openBIS for given data store.
+ *
+ * @author Izabela Adamczyk
+ */
+public class HierarchicalStorageUpdater implements IMaintenanceTask
+{
+ public static final String STOREROOT_DIR_KEY = "storeroot-dir";
+
+ public static final String HIERARCHY_ROOT_DIR_KEY = "hierarchy-root-dir";
+
+ private static final String REBUILDING_HIERARCHICAL_STORAGE = "Rebuilding hierarchical storage";
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, HierarchicalStorageUpdater.class);
+
+ private static final Logger machineLog =
+ LogFactory.getLogger(LogCategory.MACHINE, HierarchicalStorageUpdater.class);
+
+ private IEncapsulatedOpenBISService openBISService;
+
+ private String storeRoot;
+
+ private String hierarchyRoot;
+
+ public void setUp(String pluginName)
+ {
+ LogInitializer.init();
+ Properties properties = PropertyParametersUtil.loadServiceProperties();
+ storeRoot = PropertyUtils.getMandatoryProperty(properties, STOREROOT_DIR_KEY);
+ hierarchyRoot =
+ PropertyUtils.getMandatoryProperty(properties, pluginName + "."
+ + HIERARCHY_ROOT_DIR_KEY);
+ openBISService = ServiceProvider.getOpenBISService();
+ operationLog.info("Plugin initialized with: store root = " + storeRoot
+ + ", hierarchy root = " + hierarchyRoot);
+ }
+
+ public void execute()
+ {
+ rebuildHierarchy(new File(storeRoot), openBISService, new File(hierarchyRoot));
+ }
+
+ /**
+ * Refreshes the hierarchy of the data inside hierarchical storage accordingly to the database
+ * content.
+ */
+ private static void rebuildHierarchy(File storeRoot,
+ IEncapsulatedOpenBISService openBISService, File hierarchyRoot)
+ {
+ logInfo(REBUILDING_HIERARCHICAL_STORAGE);
+ Collection<SimpleDataSetInformationDTO> dataSets = openBISService.listDataSets();
+ Map<String, String> newLinkMappings =
+ convertDataToLinkMappings(storeRoot, hierarchyRoot, dataSets);
+ Set<String> toCreate = new HashSet<String>(newLinkMappings.keySet());
+ Set<String> toDelete = DataSetHierarchyHelper.extractPaths(hierarchyRoot);
+ Set<String> dontTouch = intersection(toCreate, toDelete);
+ toCreate.removeAll(dontTouch);
+ toDelete.removeAll(dontTouch);
+ removeUnnecessaryMappings(newLinkMappings, toCreate);
+ deleteObsoleteLinks(hierarchyRoot, toDelete);
+ createLinksForChangedData(newLinkMappings);
+ }
+
+ /**
+ * Extracts a {@link Map}: (target,source) from a collection of data sets.
+ */
+ private static Map<String, String> convertDataToLinkMappings(File storeRoot,
+ File hierarchyRoot, Collection<SimpleDataSetInformationDTO> dataSets)
+ {
+ Map<String, String> linkMappings = new HashMap<String, String>();
+ for (SimpleDataSetInformationDTO dataSet : dataSets)
+ {
+ File targetFile =
+ new File(hierarchyRoot, DataSetHierarchyHelper.createHierarchicalPath(dataSet));
+ File sourceFile = new File(storeRoot, dataSet.getDataSetLocation());
+ linkMappings.put(targetFile.getAbsolutePath(), sourceFile.getAbsolutePath());
+ }
+ return linkMappings;
+ }
+
+ /**
+ * Removes from the <code>linkMappings</code> map all the elements with keys not belonging to
+ * <code>keep</code> set.
+ */
+ private static void removeUnnecessaryMappings(Map<String, String> linkMappings, Set<String> keep)
+ {
+ Set<String> keys = new HashSet<String>(linkMappings.keySet());
+ for (String path : keys)
+ {
+ if (keep.contains(path) == false)
+ {
+ linkMappings.remove(path);
+ }
+ }
+ }
+
+ /**
+ * Creates a new {@link Set} containing the elements that belong to both {@link Set}s.
+ */
+ private static Set<String> intersection(Set<String> setA, Set<String> setB)
+ {
+ Set<String> toBeUntouched = new HashSet<String>(setA);
+ toBeUntouched.retainAll(setB);
+ return toBeUntouched;
+ }
+
+ /**
+ * Recursively removes from the file system files with paths defined in <code>toBeDeleted</code>
+ * {@link Set}.
+ */
+ private static void deleteObsoleteLinks(File hierarchyRoot, Set<String> toBeDeleted)
+ {
+ for (String pathToDelete : toBeDeleted)
+ {
+ File toDelete = new File(pathToDelete);
+ File parent = toDelete.getParentFile();
+ FileUtilities.deleteRecursively(toDelete);
+ while (parent != null
+ && parent.getAbsolutePath().equals(hierarchyRoot.getAbsolutePath()) == false)
+ {
+ if (parent.list().length == 0)
+ {
+ toDelete = parent;
+ parent = toDelete.getParentFile();
+ toDelete.delete();
+ } else
+ {
+ break;
+ }
+ }
+ }
+ }
+
+ /**
+ * Creates the soft links for files with paths defined in <code>linkMappings</code> {@link Map}.
+ */
+ private static void createLinksForChangedData(Map<String, String> linkMappings)
+ {
+ for (String targetPath : linkMappings.keySet())
+ {
+ File targetDir = new File(targetPath);
+ String sourcePath = linkMappings.get(targetPath);
+ File sourceFile = new File(sourcePath);
+ targetDir.mkdirs();
+ ProcessExecutionHelper.runAndLog(new SoftLinkMaker().createCommand(sourceFile,
+ targetDir), operationLog, machineLog);
+ }
+ }
+
+ private static void logInfo(String info)
+ {
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info(info);
+ }
+ }
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/DataStoreServer.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/DataStoreServer.java
index f7b4da3d8af..63f75a8c73f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/DataStoreServer.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/DataStoreServer.java
@@ -31,6 +31,5 @@ public class DataStoreServer
{
ch.systemsx.cisd.openbis.dss.generic.server.DataStoreServer.main(args);
ETLDaemon.main(args);
- HierarchicalStorageDaemon.main(args);
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataSetHierarchyHelper.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataSetHierarchyHelper.java
index b5380fb9372..56a1657c0cc 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataSetHierarchyHelper.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataSetHierarchyHelper.java
@@ -28,7 +28,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO;
*
* @author Izabela Adamczyk
*/
-class DataSetHierarchyHelper
+public class DataSetHierarchyHelper
{
/**
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/HierarchicalStorageDaemon.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/HierarchicalStorageDaemon.java
index 0f0e6c91eb9..cb9c215f52d 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/HierarchicalStorageDaemon.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/HierarchicalStorageDaemon.java
@@ -222,4 +222,5 @@ public class HierarchicalStorageDaemon
operationLog.info(info);
}
}
+
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/SoftLinkMaker.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/SoftLinkMaker.java
index c10d7f5ccbf..497327e7105 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/SoftLinkMaker.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/SoftLinkMaker.java
@@ -11,7 +11,7 @@ import ch.systemsx.cisd.base.utilities.OSUtilities;
*
* @author Izabela Adamczyk
*/
-class SoftLinkMaker
+public class SoftLinkMaker
{
private File lnExec;
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
index 73621b9b0db..e9a9bcd9751 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/ETLService.java
@@ -17,8 +17,8 @@
package ch.systemsx.cisd.openbis.generic.server;
import java.io.UnsupportedEncodingException;
-import java.util.ArrayList;
import java.util.Date;
+import java.util.ArrayList;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAO.java
index f6637f8263a..5de8b6a4755 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAO.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAO.java
@@ -16,8 +16,8 @@
package ch.systemsx.cisd.openbis.generic.server.dataaccess.db;
-import java.util.ArrayList;
import java.util.Date;
+import java.util.ArrayList;
import java.util.List;
import org.apache.log4j.Logger;
@@ -82,7 +82,7 @@ public class EventDAO extends AbstractGenericEntityDAO<EventPE> implements IEven
final DetachedCriteria criteria = DetachedCriteria.forClass(EventPE.class);
if (since != null)
{
- criteria.add(Restrictions.ge("registrationDate", since));
+ criteria.add(Restrictions.gt("registrationDate", since));
}
criteria.add(Restrictions.eq("eventType", EventType.DELETION));
criteria.add(Restrictions.eq("entityType", EntityType.DATASET));
diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAOTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAOTest.java
index 9ce069171ad..d73a4923af8 100644
--- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAOTest.java
+++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/EventDAOTest.java
@@ -139,6 +139,15 @@ public class EventDAOTest extends AbstractDAOTest
assertCorrectResult(numberOfDataSets, result);
}
+ @Test
+ public void testListDeletedDataSetsWithoutSince() throws Exception
+ {
+ saveEvent(EventType.DELETION, EntityType.DATASET, KEEP_ME + 1, BEFORE);
+ saveEvent(EventType.DELETION, EntityType.DATASET, KEEP_ME + 2, SINCE);
+ List<DeletedDataSet> result = listDataDeletionEvents(SINCE);
+ assertCorrectResult(0, result);
+ }
+
private void saveEvent(EventType eventType, EntityType entityType, String identifier, Date date)
{
String description = eventType.name() + " " + entityType.name();
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MetabolDatabaseUpdater.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MetabolDatabaseUpdater.java
new file mode 100644
index 00000000000..ccc3a360150
--- /dev/null
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/MetabolDatabaseUpdater.java
@@ -0,0 +1,162 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ch.systemsx.cisd.yeastx.etl;
+
+import java.sql.Connection;
+import java.sql.PreparedStatement;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.lang.StringEscapeUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.dbmigration.DatabaseConfigurationContext;
+import ch.systemsx.cisd.etlserver.IMaintenanceTask;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
+import ch.systemsx.cisd.yeastx.db.DBUtils;
+
+/**
+ * Maintenance task deleting from metabol database data sets which have been deleted from openbis.
+ *
+ * @author Izabela Adamczyk
+ */
+public class MetabolDatabaseUpdater implements IMaintenanceTask
+{
+
+ private static final String SYNCHRONIZATION_TABLE = "EVENTS";
+
+ private static final String SYNCHRONIZATION_TIMESTAMP = "EVENT_DATE";
+
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, MetabolDatabaseUpdater.class);
+
+ private Connection connection;
+
+ private IEncapsulatedOpenBISService openBISService;
+
+ private DatabaseConfigurationContext context;
+
+ public void setUp(String pluginName)
+ {
+ LogInitializer.init();
+ context = DBUtils.createDefaultDBContext();
+ try
+ {
+ connection = context.getDataSource().getConnection();
+ getPreviousSynchronizationDate();
+ connection.close();
+ } catch (SQLException ex)
+ {
+ throw new ConfigurationFailureException("Initialization failed", ex);
+ }
+ openBISService = ServiceProvider.getOpenBISService();
+ operationLog.info("Plugin initialized");
+ }
+
+ public void execute()
+ {
+ operationLog.info("Synchronizing data set information");
+ try
+ {
+ connection = context.getDataSource().getConnection();
+ Date previousSyncDate = getPreviousSynchronizationDate();
+ List<DeletedDataSet> deletedDataSets =
+ openBISService.listDeletedDataSets(previousSyncDate);
+ if (deletedDataSets.size() > 0)
+ {
+ boolean autoCommit = connection.getAutoCommit();
+ connection.setAutoCommit(false);
+ deleteDatasets(deletedDataSets);
+ updateSynchronizationDate(previousSyncDate, deletedDataSets);
+ connection.commit();
+ connection.setAutoCommit(autoCommit);
+ }
+ connection.close();
+ } catch (SQLException ex)
+ {
+ operationLog.error(ex);
+ }
+ }
+
+ private void deleteDatasets(List<DeletedDataSet> deletedDataSets) throws SQLException
+ {
+ connection.createStatement().execute(
+ String.format("DELETE FROM data_sets WHERE perm_id IN (%s)",
+ joinIds(deletedDataSets)));
+ }
+
+ private void updateSynchronizationDate(Date previousSyncDate, List<DeletedDataSet> deleted)
+ throws SQLException
+ {
+ Date newSynchncDate = previousSyncDate;
+ for (DeletedDataSet dds : deleted)
+ {
+ Date date = dds.getDeletionDate();
+ if (newSynchncDate == null || date.after(newSynchncDate))
+ {
+ newSynchncDate = date;
+ }
+ }
+ if (previousSyncDate == null || newSynchncDate.after(previousSyncDate))
+ {
+ PreparedStatement statement =
+ connection.prepareStatement("INSERT INTO " + SYNCHRONIZATION_TABLE + " ("
+ + SYNCHRONIZATION_TIMESTAMP + ") VALUES('" + newSynchncDate + "')");
+ statement.executeUpdate();
+ }
+ }
+
+ private String joinIds(List<DeletedDataSet> deleted)
+ {
+ StringBuilder sb = new StringBuilder();
+ for (DeletedDataSet dds : deleted)
+ {
+ if (sb.length() != 0)
+ {
+ sb.append(", ");
+ }
+ sb.append("'" + StringEscapeUtils.escapeSql(dds.getIdentifier()) + "'");
+ }
+ String ids = sb.toString();
+ return ids;
+ }
+
+ private Date getPreviousSynchronizationDate() throws SQLException
+ {
+ Date lastDeleted = null;
+ ResultSet result =
+ connection.createStatement().executeQuery(
+ "SELECT MAX(" + SYNCHRONIZATION_TIMESTAMP + ") AS "
+ + SYNCHRONIZATION_TIMESTAMP + " FROM " + SYNCHRONIZATION_TABLE);
+ while (result.next())
+ {
+ if (lastDeleted == null
+ || lastDeleted.before(result.getTimestamp(SYNCHRONIZATION_TIMESTAMP)))
+ {
+ lastDeleted = result.getTimestamp(SYNCHRONIZATION_TIMESTAMP);
+ }
+ }
+ return lastDeleted;
+ }
+}
diff --git a/rtd_yeastx/source/sql/postgresql/002/schema-002.sql b/rtd_yeastx/source/sql/postgresql/002/schema-002.sql
index 0c3fb7d9591..8e261894a37 100644
--- a/rtd_yeastx/source/sql/postgresql/002/schema-002.sql
+++ b/rtd_yeastx/source/sql/postgresql/002/schema-002.sql
@@ -162,23 +162,12 @@ CREATE TABLE FIA_CENTROIDS (
CREATE INDEX FIA_CENTROID_I_ID on FIA_CENTROIDS(FIA_MS_RUN_ID);
CREATE INDEX FIA_CENTROID_I_ID_MZ on FIA_CENTROIDS(FIA_MS_RUN_ID, MZ);
--- Table CONCENTRATIONS - WORK IN PROGRESS
+-- Table EVENTS
-CREATE TABLE CONCENTRATIONS (
- ID BIGSERIAL NOT NULL,
- EXPE_ID TECH_ID NOT NULL,
- SAMP_ID TECH_ID NOT NULL,
- DS_ID TECH_ID NOT NULL,
- COMPOUND_ID CODE NOT NULL,
- AMOUNT REAL NOT NULL
- UNIT SHORT_LABEL NOT NULL
- REGISTRATOR_ID SHORT_LABEL DEFAULT NULL,
- CALIBRATION_NOTES TEXT,
- COMMENTS TEXT,
- ACQUISITION_DATE TIMESTAMP DEFAULT NULL,
+CREATE TABLE EVENTS (
+ EVENT_DATE TIMESTAMP WITH TIME ZONE NOT NULL
);
-
GRANT SELECT ON TABLE EXPERIMENTS TO GROUP metabol_readonly;
GRANT SELECT ON TABLE SAMPLES TO GROUP metabol_readonly;
GRANT SELECT ON TABLE DATA_SETS TO GROUP metabol_readonly;
@@ -187,6 +176,7 @@ GRANT SELECT ON TABLE EIC_CHROMATOGRAMS TO GROUP metabol_readonly;
GRANT SELECT ON TABLE FIA_MS_RUNS TO GROUP metabol_readonly;
GRANT SELECT ON TABLE FIA_PROFILES TO GROUP metabol_readonly;
GRANT SELECT ON TABLE FIA_CENTROIDS TO GROUP metabol_readonly;
+GRANT SELECT ON TABLE EVENTS TO GROUP metabol_readonly;
GRANT SELECT ON SEQUENCE EXPERIMENTS_ID_SEQ TO GROUP metabol_readonly;
GRANT SELECT ON SEQUENCE SAMPLES_ID_SEQ TO GROUP metabol_readonly;
GRANT SELECT ON SEQUENCE DATA_SETS_ID_SEQ TO GROUP metabol_readonly;
@@ -204,6 +194,7 @@ GRANT ALL PRIVILEGES ON TABLE EIC_CHROMATOGRAMS TO GROUP metabol_readwrite;
GRANT ALL PRIVILEGES ON TABLE FIA_MS_RUNS TO GROUP metabol_readwrite;
GRANT ALL PRIVILEGES ON TABLE FIA_PROFILES TO GROUP metabol_readwrite;
GRANT ALL PRIVILEGES ON TABLE FIA_CENTROIDS TO GROUP metabol_readwrite;
+GRANT ALL PRIVILEGES ON TABLE EVENTS TO GROUP metabol_readwrite;
GRANT ALL PRIVILEGES ON SEQUENCE EXPERIMENTS_ID_SEQ TO GROUP metabol_readwrite;
GRANT ALL PRIVILEGES ON SEQUENCE SAMPLES_ID_SEQ TO GROUP metabol_readwrite;
GRANT ALL PRIVILEGES ON SEQUENCE DATA_SETS_ID_SEQ TO GROUP metabol_readwrite;
diff --git a/rtd_yeastx/source/sql/postgresql/migration/migration-001-002.sql b/rtd_yeastx/source/sql/postgresql/migration/migration-001-002.sql
new file mode 100644
index 00000000000..fb6816854a2
--- /dev/null
+++ b/rtd_yeastx/source/sql/postgresql/migration/migration-001-002.sql
@@ -0,0 +1,12 @@
+-----------------------------------
+-- Migration 001-002
+-----------------------------------
+
+-- Create Table EVENTS
+
+CREATE TABLE EVENTS (
+ EVENT_DATE TIMESTAMP WITH TIME ZONE NOT NULL
+);
+
+GRANT SELECT ON TABLE EVENTS TO GROUP metabol_readonly;
+GRANT ALL PRIVILEGES ON TABLE EVENTS TO GROUP metabol_readwrite;
--
GitLab