diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractDelegatingStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractDelegatingStorageProcessor.java
index 498d563d2ad89b104d7f77fc0d9a564421e99d20..1374a7198416a0b58dbba79714e5b35a1bda4b06 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractDelegatingStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractDelegatingStorageProcessor.java
@@ -92,18 +92,23 @@ abstract public class AbstractDelegatingStorageProcessor implements IStorageProc
// delegation
//
- public File storeData(final DataSetInformation dataSetInformation, final ITypeExtractor typeExtractor,
- final IMailClient mailClient, final File incomingDataSetDirectory,
- final File rootDir)
+ public File storeData(final DataSetInformation dataSetInformation,
+ final ITypeExtractor typeExtractor, final IMailClient mailClient,
+ final File incomingDataSetDirectory, final File rootDir)
{
- return delegate.storeData(dataSetInformation, typeExtractor, mailClient, incomingDataSetDirectory,
- rootDir);
+ return delegate.storeData(dataSetInformation, typeExtractor, mailClient,
+ incomingDataSetDirectory, rootDir);
}
- public UnstoreDataAction unstoreData(final File incomingDataSetDirectory,
+ public void commit()
+ {
+ delegate.commit();
+ }
+
+ public UnstoreDataAction rollback(final File incomingDataSetDirectory,
final File storedDataDirectory, Throwable exception)
{
- return delegate.unstoreData(incomingDataSetDirectory, storedDataDirectory, exception);
+ return delegate.rollback(incomingDataSetDirectory, storedDataDirectory, exception);
}
public StorageFormat getStorageFormat()
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
index 49eff0618ed38947b62c8e1b7e28bbc082eb4032..4adc58a57801ab6553b25ced7f12d806533fa340 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStorageProcessor.java
@@ -62,4 +62,9 @@ public abstract class AbstractStorageProcessor implements IStorageProcessor
{
this.storeRootDir = storeRootDirectory;
}
+
+ public void commit()
+ {
+ // do nothing
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStrorageProcessorWithUploader.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStrorageProcessorWithUploader.java
index 1a2d01888c6afdbffd2afb0e6ebdbc1da9d1bee1..68ebe2cfbac6bef7dc85dfb57a426ffae8afa4a0 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStrorageProcessorWithUploader.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/AbstractStrorageProcessorWithUploader.java
@@ -23,57 +23,71 @@ import ch.systemsx.cisd.common.mail.IMailClient;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
- * Storage processor which uses an {@link IDataSetUploader} after data set has been stored by
- * a wrapped {@link IStorageProcessor}.
- *
+ * Storage processor which uses an {@link IDataSetUploader} after data set has been stored by a
+ * wrapped {@link IStorageProcessor}.
+ *
* @author Franz-Josef Elmer
*/
-public abstract class AbstractStrorageProcessorWithUploader extends AbstractDelegatingStorageProcessor
+public abstract class AbstractStrorageProcessorWithUploader extends
+ AbstractDelegatingStorageProcessor
{
private final IDataSetUploader uploader;
- public AbstractStrorageProcessorWithUploader(IStorageProcessor processor, IDataSetUploader uploader)
+ public AbstractStrorageProcessorWithUploader(IStorageProcessor processor,
+ IDataSetUploader uploader)
{
super(processor);
this.uploader = uploader;
}
-
+
/**
- * Creates an instance with a wrapped storage processor which will be created from the
- * specified properties.
+ * Creates an instance with a wrapped storage processor which will be created from the specified
+ * properties.
*/
public AbstractStrorageProcessorWithUploader(Properties properties, IDataSetUploader uploader)
{
super(properties);
this.uploader = uploader;
}
-
+
@Override
- public File storeData(final DataSetInformation dataSetInformation, final ITypeExtractor typeExtractor,
- final IMailClient mailClient, final File incomingDataSetDirectory,
- final File rootDir)
+ public File storeData(final DataSetInformation dataSetInformation,
+ final ITypeExtractor typeExtractor, final IMailClient mailClient,
+ final File incomingDataSetDirectory, final File rootDir)
{
File storeData =
- super.storeData(dataSetInformation, typeExtractor, mailClient, incomingDataSetDirectory,
- rootDir);
+ super.storeData(dataSetInformation, typeExtractor, mailClient,
+ incomingDataSetDirectory, rootDir);
File originalData = super.tryGetProprietaryData(storeData);
uploader.upload(originalData, dataSetInformation);
return storeData;
}
@Override
- public UnstoreDataAction unstoreData(final File incomingDataSetDirectory,
+ public UnstoreDataAction rollback(final File incomingDataSetDirectory,
final File storedDataDirectory, Throwable exception)
{
- super.unstoreData(incomingDataSetDirectory, storedDataDirectory, exception);
+ try
+ {
+ super.rollback(incomingDataSetDirectory, storedDataDirectory, exception);
+ } finally
+ {
+ uploader.rollback();
+ }
logDataSetFileError(incomingDataSetDirectory, exception);
return UnstoreDataAction.LEAVE_UNTOUCHED;
}
-
+
+ @Override
+ public void commit()
+ {
+ super.commit();
+ uploader.commit();
+ }
+
/**
* Logs an error for the specified data set and exception.
*/
protected abstract void logDataSetFileError(File incomingDataSetDirectory, Throwable exception);
-
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/BDSStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/BDSStorageProcessor.java
index df41677e5154522061001a1b9d532c667d000399..78c12ef7ee1aa1584d2e4d99a12559ce7fbb4148 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/BDSStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/BDSStorageProcessor.java
@@ -471,7 +471,7 @@ public final class BDSStorageProcessor extends AbstractStorageProcessor implemen
return dataStructureDir;
}
- public UnstoreDataAction unstoreData(File incomingDataSetDirectory, File storedDataDirectory,
+ public UnstoreDataAction rollback(File incomingDataSetDirectory, File storedDataDirectory,
Throwable exception)
{
unstoreData(incomingDataSetDirectory, storedDataDirectory);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/CifexStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/CifexStorageProcessor.java
index 1557db2cce46d4bfd3206557c4750374c2b3d13c..6b5a23dea5254b484944f1f866e77930756002af 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/CifexStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/CifexStorageProcessor.java
@@ -82,7 +82,7 @@ public class CifexStorageProcessor extends AbstractDelegatingStorageProcessor
}
@Override
- public UnstoreDataAction unstoreData(File incomingDataSetDirectory, File storedDataDirectory,
+ public UnstoreDataAction rollback(File incomingDataSetDirectory, File storedDataDirectory,
Throwable exception)
{
if (dirDeleted)
@@ -91,7 +91,7 @@ public class CifexStorageProcessor extends AbstractDelegatingStorageProcessor
FileOperations.getMonitoredInstanceForCurrentThread().moveToDirectory(fileToMove,
dirToRestore);
}
- super.unstoreData(incomingDataSetDirectory, storedDataDirectory, exception);
+ super.rollback(incomingDataSetDirectory, storedDataDirectory, exception);
return UnstoreDataAction.DELETE;
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
index cde68a619e58ca1374c35dd7b5f4e2e9a324db13..970cee3860378875a8cdbbda7b762bdae74e27dd 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessor.java
@@ -47,9 +47,9 @@ public class DefaultStorageProcessor extends AbstractStorageProcessor
// AbstractStorageProcessor
//
- public final File storeData(final DataSetInformation dataSetInformation, final ITypeExtractor typeExtractor,
- final IMailClient mailClient, final File incomingDataSetDirectory,
- final File rootDir)
+ public final File storeData(final DataSetInformation dataSetInformation,
+ final ITypeExtractor typeExtractor, final IMailClient mailClient,
+ final File incomingDataSetDirectory, final File rootDir)
{
checkParameters(incomingDataSetDirectory, rootDir);
File originalDir = getOriginalDirectory(rootDir);
@@ -67,7 +67,7 @@ public class DefaultStorageProcessor extends AbstractStorageProcessor
return rootDir;
}
- public UnstoreDataAction unstoreData(final File incomingDataSetDirectory,
+ public UnstoreDataAction rollback(final File incomingDataSetDirectory,
final File storedDataDirectory, Throwable exception)
{
checkParameters(incomingDataSetDirectory, storedDataDirectory);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DelegatingStorageProcessorWithDropbox.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DelegatingStorageProcessorWithDropbox.java
index 9208a39e94909388541caa43a0d6051209bc4327..45398c24e2e85012a6ad4dbc047c31a2fe5f6df8 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/DelegatingStorageProcessorWithDropbox.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/DelegatingStorageProcessorWithDropbox.java
@@ -79,10 +79,10 @@ public class DelegatingStorageProcessorWithDropbox extends
}
@Override
- public UnstoreDataAction unstoreData(final File incomingDataSetDirectory,
+ public UnstoreDataAction rollback(final File incomingDataSetDirectory,
final File storedDataDirectory, Throwable exception)
{
dropboxHandler.undoLastOperation();
- return super.unstoreData(incomingDataSetDirectory, storedDataDirectory, exception);
+ return super.rollback(incomingDataSetDirectory, storedDataDirectory, exception);
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetUploader.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetUploader.java
index 940e4235344e729e9d554d728a5dec74df0c5a8b..58fd4d5c15bdd32627fc1818fd6ec5ad92ceb1a9 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetUploader.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IDataSetUploader.java
@@ -22,15 +22,18 @@ import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
/**
- *
- *
* @author Franz-Josef Elmer
*/
public interface IDataSetUploader
{
/** uploads files with recognized extensions to the additional database */
- public void upload(File dataSet, DataSetInformation dataSetInformation)
+ void upload(File dataSet, DataSetInformation dataSetInformation)
throws EnvironmentFailureException;
+ /** commits the results of the last call of {@link #upload} */
+ void commit();
+
+ /** rollbacks the results of the last call of {@link #upload} */
+ void rollback();
}
\ No newline at end of file
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStorageProcessor.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStorageProcessor.java
index 05a1a461c3d197f82df7f94a539ef2501fd349ba..1fa1a17e3b3e0580c325de44641ca632ab971392 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStorageProcessor.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/IStorageProcessor.java
@@ -44,20 +44,31 @@ public interface IStorageProcessor extends IStoreRootDirectoryHolder
* Do not try/catch exceptions that could occur here. Preferably let the upper layer handle
* them.
* </p>
+ *
* @param dataSetInformation Information about the data set.
* @param typeExtractor the {@link ITypeExtractor} implementation.
* @param mailClient mail client.
* @param incomingDataSetDirectory folder to store. Do not remove it after the implementation
* has finished processing. {@link TransferredDataSetHandler} takes care of this.
* @param rootDir directory to whom the data will be stored.
- *
* @return folder which contains the stored data. This folder <i>must</i> be below the
* <var>rootDir</var>. Never returns <code>null</code> but prefers to throw an exception
* in case of unexpected behavior.
*/
- public File storeData(final DataSetInformation dataSetInformation, final ITypeExtractor typeExtractor,
- final IMailClient mailClient, final File incomingDataSetDirectory,
- final File rootDir);
+ public File storeData(final DataSetInformation dataSetInformation,
+ final ITypeExtractor typeExtractor, final IMailClient mailClient,
+ final File incomingDataSetDirectory, final File rootDir);
+
+ /**
+ * Commits the changes done by the recent {@link #storeData} call if the dataset has been also
+ * successfully registered openBIS.
+ * <p>
+ * This operation is useful when the storage processor adds the data to an additional database.
+ * If all the storage processor operations are done on the file system, the implementation of
+ * this method will be usually empty.
+ * </p>
+ */
+ public void commit();
/**
* Instructs the dataset handler what to do with the data in incoming directory if there was an
@@ -73,13 +84,16 @@ public interface IStorageProcessor extends IStoreRootDirectoryHolder
* leave the data in the incoming directory
*/
LEAVE_UNTOUCHED,
+ /**
+ * delete the data from the incoming directory
+ */
DELETE
}
/**
* Performs a rollback of
- * {@link #storeData(DataSetInformation, ITypeExtractor, IMailClient, File, File)} The
- * data created in <code>directory</code> will also be removed.
+ * {@link #storeData(DataSetInformation, ITypeExtractor, IMailClient, File, File)} The data
+ * created in <code>directory</code> will also be removed.
* <p>
* Call to this method is safe as implementations should try/catch exceptions that could occur
* here.
@@ -90,7 +104,7 @@ public interface IStorageProcessor extends IStoreRootDirectoryHolder
* @param exception an exception which has caused that the unstore operation has to be performed
* @return an instruction what to do with the data in incoming directory
*/
- public UnstoreDataAction unstoreData(final File incomingDataSetDirectory,
+ public UnstoreDataAction rollback(final File incomingDataSetDirectory,
final File storedDataDirectory, Throwable exception);
/**
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
index f45881bea41b9972af66df5f33fec22532360563..28787adf0f2f7f0178d72a63dda278086b9a7822 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandler.java
@@ -143,27 +143,28 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
private final IDataSetValidator dataSetValidator;
/**
- * @param dataSetValidator
+ * @param dataSetValidator
* @param useIsFinishedMarkerFile if true, file/directory is processed when a marker file for it
* appears. Otherwise processing starts if the file/directory is not modified for a
* certain amount of time (so called "quiet period").
*/
public TransferredDataSetHandler(String dssCode, final IETLServerPlugin plugin,
final IEncapsulatedOpenBISService limsService, final Properties mailProperties,
- IDataSetValidator dataSetValidator, final boolean notifySuccessfulRegistration, boolean useIsFinishedMarkerFile,
- boolean deleteUnidentified)
+ IDataSetValidator dataSetValidator, final boolean notifySuccessfulRegistration,
+ boolean useIsFinishedMarkerFile, boolean deleteUnidentified)
{
this(dssCode, plugin.getStorageProcessor(), plugin, limsService, new MailClient(
- mailProperties), dataSetValidator, notifySuccessfulRegistration, useIsFinishedMarkerFile,
- deleteUnidentified);
+ mailProperties), dataSetValidator, notifySuccessfulRegistration,
+ useIsFinishedMarkerFile, deleteUnidentified);
}
TransferredDataSetHandler(String dssCode,
final IStoreRootDirectoryHolder storeRootDirectoryHolder,
final IETLServerPlugin plugin, final IEncapsulatedOpenBISService limsService,
- final IMailClient mailClient, IDataSetValidator dataSetValidator, final boolean notifySuccessfulRegistration,
- boolean useIsFinishedMarkerFile, boolean deleteUnidentified)
+ final IMailClient mailClient, IDataSetValidator dataSetValidator,
+ final boolean notifySuccessfulRegistration, boolean useIsFinishedMarkerFile,
+ boolean deleteUnidentified)
{
assert dssCode != null : "Unspecified data store code";
@@ -438,6 +439,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
operationLog.error("Cannot delete '" + incomingDataSetFile.getAbsolutePath()
+ "'.");
}
+
clean();
return Collections.singletonList(dataSetInformation);
} catch (final HighLevelException ex)
@@ -501,7 +503,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
throw (Error) throwable;
}
UnstoreDataAction action =
- storageProcessor.unstoreData(incomingDataSetFile, baseDirectoryHolder
+ storageProcessor.rollback(incomingDataSetFile, baseDirectoryHolder
.getBaseDirectory(), throwable);
if (stopped == false)
{
@@ -522,7 +524,8 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
}
} else if (action == UnstoreDataAction.DELETE)
{
- FileUtilities.deleteRecursively(incomingDataSetFile, new Log4jSimpleLogger(operationLog));
+ FileUtilities.deleteRecursively(incomingDataSetFile, new Log4jSimpleLogger(
+ operationLog));
}
}
}
@@ -569,6 +572,7 @@ public final class TransferredDataSetHandler implements IPathHandler, ISelfTesta
{
getRegistrationLock().unlock();
}
+ storageProcessor.commit();
} finally
{
fileOperations.delete(markerFile);
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/BDSStorageProcessorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/BDSStorageProcessorTest.java
index 25a9ee2f09f6cf11f9d8398e70192667a0f923de..2ea26a76a62e87f97fcc0f5db21c5b708b0821da 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/BDSStorageProcessorTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/BDSStorageProcessorTest.java
@@ -418,7 +418,7 @@ public final class BDSStorageProcessorTest extends AbstractFileSystemTestCase
incomingDirectoryData, storeRootDir);
assertEquals(true, dataStore.isDirectory());
assertEquals(false, incomingDirectoryData.exists());
- storageAdapter.unstoreData(incomingDirectoryData, storeRootDir, null);
+ storageAdapter.rollback(incomingDirectoryData, storeRootDir, null);
assertEquals(false, dataStore.exists());
assertEquals(true, incomingDirectoryData.isDirectory());
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessorTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessorTest.java
index 324630df94ae51b0a6325542bf0d5d7267e2e854..76c9cdc8a0d124b8954bbce8b827cb3036e34b9c 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessorTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/DefaultStorageProcessorTest.java
@@ -97,7 +97,7 @@ public final class DefaultStorageProcessorTest extends AbstractFileSystemTestCas
final DefaultStorageProcessor storageProcessor = createStorageProcessor();
try
{
- storageProcessor.unstoreData(null, null, null);
+ storageProcessor.rollback(null, null, null);
fail("Null values not accepted");
} catch (final AssertionError e)
{
@@ -112,7 +112,7 @@ public final class DefaultStorageProcessorTest extends AbstractFileSystemTestCas
root);
assertEquals(true, storeData.exists());
assertEquals(false, incomingDataSetDirectory.exists());
- storageProcessor.unstoreData(incomingDataSetDirectory, root, null);
+ storageProcessor.rollback(incomingDataSetDirectory, root, null);
assertEquals(true, incomingDataSetDirectory.exists());
assertEquals("hi", FileUtilities.loadToString(readMeFile).trim());
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
index 8447ae63796bc387b5fc14385263f90554a095fa..e02eb26022e96d14e7d5ba5db7d7cf2740216ba9 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/TransferredDataSetHandlerTest.java
@@ -542,6 +542,7 @@ public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestC
data1, baseDir);
final File finalDataSetPath = new File(baseDir, DATA1_NAME);
will(returnValue(finalDataSetPath));
+ one(storageProcessor).commit();
}
});
final LogMonitoringAppender appender =
@@ -584,6 +585,7 @@ public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestC
data1, baseDir);
final File finalDataSetPath = new File(baseDir, DATA1_NAME);
will(returnValue(finalDataSetPath));
+ one(storageProcessor).commit();
}
});
final LogMonitoringAppender appender =
@@ -682,7 +684,7 @@ public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestC
new UserFailureException("Could store data by storage processor");
will(throwException(exception));
- one(storageProcessor).unstoreData(folder, baseDir, exception);
+ one(storageProcessor).rollback(folder, baseDir, exception);
will(returnValue(UnstoreDataAction.MOVE_TO_ERROR));
allowing(typeExtractor).getLocatorType(folder);
@@ -746,7 +748,7 @@ public final class TransferredDataSetHandlerTest extends AbstractFileSystemTestC
new EnvironmentFailureException("Could not register data set folder");
will(throwException(exception));
- one(storageProcessor).unstoreData(folder, baseDir, exception);
+ one(storageProcessor).rollback(folder, baseDir, exception);
one(storageProcessor).getStorageFormat();
}
});
diff --git a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ProtXMLUploader.java b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ProtXMLUploader.java
index 59f928371f45acbe53e6c86e40aa4e8f8642e626..08a6c839369afab7ccf889e8568ed20ec47a9eed 100644
--- a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ProtXMLUploader.java
+++ b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ProtXMLUploader.java
@@ -42,7 +42,7 @@ import ch.systemsx.cisd.openbis.etlserver.phosphonetx.dto.ProteinSummary;
public class ProtXMLUploader implements IDataSetUploader
{
private static final String VALIDATING_XML = "validating-xml";
-
+
private static final String DATABASE_ENGINE = "database.engine";
private static final String DATABASE_URL_HOST_PART = "database.url-host-part";
@@ -76,6 +76,8 @@ public class ProtXMLUploader implements IDataSetUploader
private final DataSource dataSource;
+ private ResultDataSetUploader currentResultDataSetUploader;
+
public ProtXMLUploader(Properties properties, IEncapsulatedOpenBISService openbisService)
{
dataSource = createDataSource(properties);
@@ -95,8 +97,13 @@ public class ProtXMLUploader implements IDataSetUploader
+ (System.currentTimeMillis() - time) + " msec");
}
time = System.currentTimeMillis();
- ResultDataSetUploader upLoader = createUploader();
- upLoader.upload(dataSetInformation, summary);
+ if (currentResultDataSetUploader != null)
+ {
+ throw new IllegalStateException(
+ "The previous transaction has been neither commited nor rollbacked.");
+ }
+ currentResultDataSetUploader = createUploader();
+ currentResultDataSetUploader.upload(dataSetInformation, summary);
if (operationLog.isInfoEnabled())
{
operationLog.info("Feeding result database took " + (System.currentTimeMillis() - time)
@@ -104,6 +111,18 @@ public class ProtXMLUploader implements IDataSetUploader
}
}
+ public void commit()
+ {
+ currentResultDataSetUploader.commit();
+ currentResultDataSetUploader = null;
+ }
+
+ public void rollback()
+ {
+ currentResultDataSetUploader.rollback();
+ currentResultDataSetUploader = null;
+ }
+
protected ResultDataSetUploader createUploader()
{
Connection connection;
@@ -134,5 +153,4 @@ public class ProtXMLUploader implements IDataSetUploader
}
throw new UserFailureException("No *prot.xml file found in data set '" + dataSet + "'.");
}
-
}
diff --git a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ResultDataSetUploader.java b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ResultDataSetUploader.java
index 341800e89773540a5d639cbf3aa173741286ed4b..f1dab90537912830a7cb38ee7efb55c6db4d58e2 100644
--- a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ResultDataSetUploader.java
+++ b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ResultDataSetUploader.java
@@ -80,7 +80,8 @@ class ResultDataSetUploader extends AbstractHandler
{
try
{
- Experiment experiment = getOrCreateExperiment(dataSetInfo.tryToGetExperiment().getPermId());
+ Experiment experiment =
+ getOrCreateExperiment(dataSetInfo.tryToGetExperiment().getPermId());
ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample dataSetInfoSample =
dataSetInfo.tryToGetSample();
Sample sample = null;
@@ -89,7 +90,8 @@ class ResultDataSetUploader extends AbstractHandler
{
sample = getOrCreateSample(experiment, dataSetInfoSample.getPermId());
Group group = dataSetInfoSample.getGroup();
- groupIdentifier = new GroupIdentifier(group.getInstance().getCode(), group.getCode());
+ groupIdentifier =
+ new GroupIdentifier(group.getInstance().getCode(), group.getCode());
} else
{
ExperimentIdentifier experimentIdentifier = dataSetInfo.getExperimentIdentifier();
@@ -107,26 +109,45 @@ class ResultDataSetUploader extends AbstractHandler
DataSet ds =
getOrCreateDataSet(experiment, sample, database, dataSetInfo.getDataSetCode());
addToDatabase(ds, experiment, groupIdentifier, summary);
- if (errorMessages.length() == 0)
- {
- connection.commit();
- } else
- {
- throw UserFailureException.fromTemplate(
- "Following errors occurred while uploading protein information"
- + " to the dataset database from the dataset '%s': %s" + " ",
- dataSetInfo.getDataSetCode(), errorMessages.toString());
- }
} catch (Throwable throwable)
{
try
{
connection.rollback();
- throw CheckedExceptionTunnel.wrapIfNecessary(throwable);
} catch (SQLException ex)
{
- throw CheckedExceptionTunnel.wrapIfNecessary(ex);
}
+ throw CheckedExceptionTunnel.wrapIfNecessary(throwable);
+ }
+ if (errorMessages.length() != 0)
+ {
+ rollback();
+ throw UserFailureException.fromTemplate(
+ "Following errors occurred while uploading protein information"
+ + " to the dataset database from the dataset '%s': %s" + " ",
+ dataSetInfo.getDataSetCode(), errorMessages.toString());
+ }
+ }
+
+ public void rollback()
+ {
+ try
+ {
+ connection.rollback();
+ } catch (SQLException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void commit()
+ {
+ try
+ {
+ connection.commit();
+ } catch (SQLException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
}
}
@@ -151,8 +172,8 @@ class ResultDataSetUploader extends AbstractHandler
return database;
}
- private DataSet getOrCreateDataSet(Experiment experiment, Sample sampleOrNull, Database database,
- String dataSetPermID)
+ private DataSet getOrCreateDataSet(Experiment experiment, Sample sampleOrNull,
+ Database database, String dataSetPermID)
{
DataSet dataSet = dao.tryToGetDataSetByPermID(dataSetPermID);
if (dataSet == null)
@@ -186,8 +207,8 @@ class ResultDataSetUploader extends AbstractHandler
return experiment;
}
- private void addToDatabase(DataSet dataSet, Experiment experiment, GroupIdentifier groupIdentifier,
- ProteinSummary summary)
+ private void addToDatabase(DataSet dataSet, Experiment experiment,
+ GroupIdentifier groupIdentifier, ProteinSummary summary)
{
long dataSetID = dataSet.getId();
Long databaseID = dataSet.getDatabaseID();
@@ -357,4 +378,3 @@ class ResultDataSetUploader extends AbstractHandler
throw new UserFailureException("Missing Protein Prophet details.");
}
}
-
diff --git a/rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ResultDataSetUploaderTest.java b/rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ResultDataSetUploaderTest.java
index b3710227262b70926a79ef3d40f73814fed55748..827d7e95028fb3d5a9ebcc7e0f0060d9fcf8e8a8 100644
--- a/rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ResultDataSetUploaderTest.java
+++ b/rtd_phosphonetx/sourceTest/java/ch/systemsx/cisd/openbis/etlserver/phosphonetx/ResultDataSetUploaderTest.java
@@ -177,7 +177,6 @@ public class ResultDataSetUploaderTest extends AssertJUnit
will(returnValue(dataSet));
}
});
- prepareForCommit();
uploader.upload(createDataSetInfo(), createProteinSummary());
@@ -188,7 +187,6 @@ public class ResultDataSetUploaderTest extends AssertJUnit
public void testNoProteinsCreateExperimentSampleDatabaseAndDataSet()
{
prepareForCreatingExperimentSampleDatabaseAndDataSet();
- prepareForCommit();
uploader.upload(createDataSetInfo(), createProteinSummary());
@@ -199,7 +197,6 @@ public class ResultDataSetUploaderTest extends AssertJUnit
public void testEmptyProteinGroup()
{
prepareForCreatingExperimentSampleDatabaseAndDataSet();
- prepareForCommit();
ProteinSummary summary = createProteinSummary();
summary.getProteinGroups().add(createProteinGroup());
@@ -218,7 +215,6 @@ public class ResultDataSetUploaderTest extends AssertJUnit
prepareForCreatingIdentifiedProtein(a1, false);
ProteinAnnotation a2 = createAnnotation(UNIPROT_ID2, PROTEIN_NAME2, SEQUENCE2);
prepareForCreatingIdentifiedProtein(a2, true);
- prepareForCommit();
ProteinSummary summary = createProteinSummary();
Protein p1 = createProtein(probability, a1, a2);
@@ -226,7 +222,6 @@ public class ResultDataSetUploaderTest extends AssertJUnit
summary.getProteinGroups().add(createProteinGroup(p1, new Protein()));
uploader.upload(createDataSetInfo(), summary);
-
context.assertIsSatisfied();
}
@@ -282,7 +277,7 @@ public class ResultDataSetUploaderTest extends AssertJUnit
prepareForCommit();
uploader.upload(createDataSetInfo(), summary);
-
+ uploader.commit();
context.assertIsSatisfied();
}
@@ -301,8 +296,8 @@ public class ResultDataSetUploaderTest extends AssertJUnit
final SampleIdentifier sampleIdentifier =
new SampleIdentifier(groupIdentifier, CELL_LYSATE1);
final ListSamplesByPropertyCriteria criteria =
- new ListSamplesByPropertyCriteria(AbundanceHandler.MZXML_FILENAME,
- CELL_LYSATE1, GROUP_CODE, null);
+ new ListSamplesByPropertyCriteria(AbundanceHandler.MZXML_FILENAME, CELL_LYSATE1,
+ GROUP_CODE, null);
context.checking(new Expectations()
{
{
@@ -335,8 +330,8 @@ public class ResultDataSetUploaderTest extends AssertJUnit
} catch (UserFailureException ex)
{
AssertionUtil.assertContains("Protein '" + PROTEIN_NAME1
- + "' has an abundance value for an unidentified sample: "
- + CELL_LYSATE1, ex.getMessage());
+ + "' has an abundance value for an unidentified sample: " + CELL_LYSATE1, ex
+ .getMessage());
}
context.assertIsSatisfied();
@@ -350,7 +345,6 @@ public class ResultDataSetUploaderTest extends AssertJUnit
prepareForCreatingProtein(probability);
ProteinAnnotation a1 = createAnnotation(UNIPROT_ID1, PROTEIN_NAME1, SEQUENCE1);
prepareForCreatingIdentifiedProtein(a1, false);
- prepareForCommit();
ProteinSummary summary = createProteinSummary();
Protein p1 = createProtein(probability, a1);
@@ -380,7 +374,6 @@ public class ResultDataSetUploaderTest extends AssertJUnit
prepareForCreatingProtein(probability);
ProteinAnnotation a1 = createAnnotation(UNIPROT_ID1, PROTEIN_NAME1, SEQUENCE1);
prepareForCreatingIdentifiedProtein(a1, false);
- prepareForCommit();
ProteinSummary summary = createProteinSummary();
Protein p1 = createProtein(probability, a1);
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/AbstractDatasetLoader.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/AbstractDatasetLoader.java
new file mode 100644
index 0000000000000000000000000000000000000000..a360f400414fcd894be90253e215f479b97f6565
--- /dev/null
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/AbstractDatasetLoader.java
@@ -0,0 +1,82 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.yeastx.db;
+
+import org.springframework.dao.DataAccessException;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+
+/**
+ * Abstract class for dataset loaders. Provides commit and rollback functionality and creation of
+ * the dataset records (without the detail data).
+ *
+ * @author Tomasz Pylak
+ */
+abstract public class AbstractDatasetLoader implements IDatasetLoader
+{
+ abstract protected IGenericDAO getDao();
+
+ // if false transaction has to be commited or rollbacked before the next dataset will be created
+ protected boolean isTransactionCompleted = true;
+
+ /**
+ * Cannot be called twice in a row if {@link #commit()} or {@link #rollback()} has not been
+ * called in between.
+ */
+ protected void createDataSet(DMDataSetDTO dataSet)
+ {
+ if (isTransactionCompleted == false)
+ {
+ throw new IllegalStateException(
+ "The previous transaction of uploading a dataset has been neither commited nor rollbacked.");
+ }
+ DBUtils.createDataSet(getDao(), dataSet);
+ isTransactionCompleted = false;
+ }
+
+ protected void rollbackAndRethrow(Throwable exception) throws Error
+ {
+ try
+ {
+ rollback();
+ } catch (DataAccessException ex)
+ {
+ // Avoid this exception shadowing the original exception.
+ }
+ throw CheckedExceptionTunnel.wrapIfNecessary(exception);
+ }
+
+ public void commit()
+ {
+ try
+ {
+ getDao().close(true);
+ } catch (Throwable th)
+ {
+ rollbackAndRethrow(th);
+ } finally
+ {
+ isTransactionCompleted = true;
+ }
+ }
+
+ public void rollback()
+ {
+ isTransactionCompleted = true;
+ DBUtils.rollbackAndClose(getDao());
+ }
+}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/DBUtils.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/DBUtils.java
index e83ded7eebf9d7cb6a147f8e2d908f32fd4651e6..e88b13a60eb31ea4c451d433b2ccd877a2d92889 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/DBUtils.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/DBUtils.java
@@ -86,7 +86,8 @@ public class DBUtils
/**
* Creates the data set based on the information given in <var>dataSet</var>. The sample and
* experiment of the data set may already exist in the database. If they don't, they are created
- * as well.
+ * as well.<br>
+ * Sets sample, experiment and dataset id fields of the specified dataset.
* <p>
* NOTE: Code responsible for trying to get sample and experiment from the DB and creating them
* if they don't exist is in synchronized block and uses currently opened transaction. Then the
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/IDatasetLoader.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/IDatasetLoader.java
new file mode 100644
index 0000000000000000000000000000000000000000..1e1fd4d9edff58132c1e27d2a973ba83b165a68d
--- /dev/null
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/IDatasetLoader.java
@@ -0,0 +1,40 @@
+/*
+ * Copyright 2009 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.yeastx.db;
+
+import java.io.File;
+import java.sql.SQLException;
+
+/**
+ * @author Tomasz Pylak
+ */
+public interface IDatasetLoader
+{
+
+ /**
+ * Uploads files with recognized extensions to the additional database.<br>
+ * If the exception is thrown than all the operations are rollbacked automatically and the next
+ * upload can take place without calling {@link #rollback()} or {@link #commit()} explicitly.
+ */
+ void upload(final File file, final DMDataSetDTO dataSet) throws SQLException;
+
+ /** Commits the previous {@link #upload} */
+ void commit();
+
+ /** Rollbacks the previous {@link #upload} */
+ void rollback();
+}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/eicml/EICML2Database.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/eicml/EICML2Database.java
index 49d8a640634756387ab8623486f4931ee18c15c2..41c74a08be07a7b3f0554aa90bb86eb72461969f 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/eicml/EICML2Database.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/eicml/EICML2Database.java
@@ -17,23 +17,16 @@
package ch.systemsx.cisd.yeastx.eicml;
import java.io.File;
-import java.io.IOException;
-import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
import javax.sql.DataSource;
-import javax.xml.parsers.ParserConfigurationException;
import net.lemnik.eodsql.QueryTool;
-import net.lemnik.eodsql.TransactionQuery;
-import org.springframework.dao.DataAccessException;
-import org.xml.sax.SAXException;
-
-import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
-import ch.systemsx.cisd.yeastx.db.DBUtils;
+import ch.systemsx.cisd.yeastx.db.AbstractDatasetLoader;
import ch.systemsx.cisd.yeastx.db.DMDataSetDTO;
+import ch.systemsx.cisd.yeastx.db.IGenericDAO;
import ch.systemsx.cisd.yeastx.eicml.EICMLParser.IChromatogramObserver;
import ch.systemsx.cisd.yeastx.eicml.EICMLParser.IMSRunObserver;
@@ -42,7 +35,7 @@ import ch.systemsx.cisd.yeastx.eicml.EICMLParser.IMSRunObserver;
*
* @author Bernd Rinn
*/
-public class EICML2Database
+public class EICML2Database extends AbstractDatasetLoader
{
private final static int CHROMATOGRAM_BATCH_SIZE = 100;
@@ -67,16 +60,14 @@ public class EICML2Database
/**
* Method for uploading an <var>eicMLFile</var> to the database.
*/
- public void uploadEicMLFile(final File eicMLFile, final DMDataSetDTO dataSet)
+ public void upload(final File eicMLFile, final DMDataSetDTO dataSet)
{
final long[] eicMLId = new long[1];
final List<ChromatogramDTO> chromatograms =
new ArrayList<ChromatogramDTO>(CHROMATOGRAM_BATCH_SIZE);
- TransactionQuery transaction = null;
try
{
- transaction = dao;
- DBUtils.createDataSet(dao, dataSet);
+ createDataSet(dataSet);
new EICMLParser(eicMLFile.getPath(), new IMSRunObserver()
{
public void observe(EICMSRunDTO run)
@@ -98,35 +89,15 @@ public class EICML2Database
}
});
addChromatograms(dao, eicMLId[0], chromatograms, 1);
- transaction.close(true);
} catch (Throwable th)
{
- try
- {
- DBUtils.rollbackAndClose(transaction);
- } catch (DataAccessException ex)
- {
- // Avoid this exception shadowing the original exception.
- }
- throw CheckedExceptionTunnel.wrapIfNecessary(th);
+ rollbackAndRethrow(th);
}
}
- public static void main(String[] args) throws ParserConfigurationException, SAXException,
- IOException, SQLException
+ @Override
+ protected IGenericDAO getDao()
{
- final long start = System.currentTimeMillis();
- final EICML2Database eicML2Database =
- new EICML2Database(DBUtils.createDefaultDBContext().getDataSource());
- final String dir = args[0];
- int permId = 0;
- for (String f : new File(dir).list(new EICMLFilenameFilter()))
- {
- eicML2Database.uploadEicMLFile(new File(dir, f), new DMDataSetDTO(Integer
- .toString(++permId), "sample1", "the sample name", "experiment1",
- "the experiment name"));
- }
- System.out.println((System.currentTimeMillis() - start) / 1000.0);
+ return dao;
}
-
}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/ML2DatabaseUploader.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/ML2DatabaseUploader.java
index e785facc760ac617d48c063a51df46c9eb173c12..920bd6df5addcfdd2e3c0c44596e22bf1b9f0460 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/ML2DatabaseUploader.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/ML2DatabaseUploader.java
@@ -21,6 +21,8 @@ import java.sql.SQLException;
import java.util.List;
import java.util.Properties;
+import javax.sql.DataSource;
+
import org.apache.commons.io.FilenameUtils;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
@@ -34,6 +36,7 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
import ch.systemsx.cisd.yeastx.db.DBUtils;
import ch.systemsx.cisd.yeastx.db.DMDataSetDTO;
+import ch.systemsx.cisd.yeastx.db.IDatasetLoader;
import ch.systemsx.cisd.yeastx.eicml.EICML2Database;
import ch.systemsx.cisd.yeastx.fiaml.FIAML2Database;
import ch.systemsx.cisd.yeastx.mzxml.MzXml2Database;
@@ -51,18 +54,22 @@ public class ML2DatabaseUploader implements IDataSetUploader
private static final String DATABASE_PROPERTIES_PREFIX = "database.";
- private final EICML2Database eicML2Database;
+ private final IDatasetLoader eicML2Database;
- private final FIAML2Database fiaML2Database;
+ private final IDatasetLoader fiaML2Database;
- private final QuantML2Database quantML2Database;
+ private final IDatasetLoader quantML2Database;
- private final MzXml2Database mzXml2Database;
+ private final IDatasetLoader mzXml2Database;
private final String uniqueSampleNamePropertyCode;
private final String uniqueExperimentNamePropertyCode;
+ // ------------
+
+ private IDatasetLoader currentTransaction;
+
public ML2DatabaseUploader(Properties properties)
{
final Properties dbProps =
@@ -71,10 +78,12 @@ public class ML2DatabaseUploader implements IDataSetUploader
(dbProps.isEmpty() ? DBUtils.createDefaultDBContext() : BeanUtils.createBean(
DatabaseConfigurationContext.class, dbProps));
DBUtils.init(dbContext);
- this.eicML2Database = new EICML2Database(dbContext.getDataSource());
- this.fiaML2Database = new FIAML2Database(dbContext.getDataSource());
- this.quantML2Database = new QuantML2Database(dbContext.getDataSource());
- this.mzXml2Database = new MzXml2Database(dbContext.getDataSource());
+ DataSource dataSource = dbContext.getDataSource();
+ this.eicML2Database = new EICML2Database(dataSource);
+ this.fiaML2Database = new FIAML2Database(dataSource);
+ this.quantML2Database = new QuantML2Database(dataSource);
+ this.mzXml2Database = new MzXml2Database(dataSource);
+ this.currentTransaction = null;
this.uniqueExperimentNamePropertyCode =
DatasetMappingResolver.getUniqueExperimentNamePropertyCode(properties);
this.uniqueSampleNamePropertyCode =
@@ -87,7 +96,17 @@ public class ML2DatabaseUploader implements IDataSetUploader
{
try
{
- uoloadFile(dataSet, dataSetInformation);
+ if (currentTransaction != null)
+ {
+ throw new IllegalStateException(
+ "Current transaction has been neither commited nor rollbacked.");
+ }
+ this.currentTransaction = tryGetDatasetUploader(dataSet, dataSetInformation);
+ if (currentTransaction != null)
+ {
+ DMDataSetDTO openbisBacklink = createBacklink(dataSetInformation);
+ currentTransaction.upload(dataSet, openbisBacklink);
+ }
} catch (SQLException e)
{
throw EnvironmentFailureException
@@ -98,30 +117,55 @@ public class ML2DatabaseUploader implements IDataSetUploader
}
}
- private void uoloadFile(File dataSet, DataSetInformation dataSetInformation)
+ private IDatasetLoader tryGetDatasetUploader(File dataSet, DataSetInformation dataSetInformation)
throws SQLException
{
- DMDataSetDTO openbisBacklink = createBacklink(dataSetInformation);
String extension = getExtension(dataSet);
if (extension.equalsIgnoreCase(ConstantsYeastX.FIAML_EXT))
{
- fiaML2Database.uploadFiaMLFile(dataSet, openbisBacklink);
+ return fiaML2Database;
} else if (extension.equalsIgnoreCase(ConstantsYeastX.EICML_EXT))
{
- eicML2Database.uploadEicMLFile(dataSet, openbisBacklink);
+ return eicML2Database;
} else if (extension.equalsIgnoreCase(ConstantsYeastX.QUANTML_EXT))
{
- quantML2Database.uploadQuantMLFile(dataSet, openbisBacklink);
+ return quantML2Database;
} else if (extension.equalsIgnoreCase(ConstantsYeastX.MZXML_EXT))
{
DataSetInformationYeastX info = (DataSetInformationYeastX) dataSetInformation;
if (info.getConversion() == MLConversionType.NONE)
{
- mzXml2Database.uploadFile(dataSet, openbisBacklink);
+ return mzXml2Database;
}
- } else
+ }
+ return null;
+ }
+
+ public void commit()
+ {
+ try
{
- // do nothing
+ if (currentTransaction != null)
+ {
+ currentTransaction.commit();
+ }
+ } finally
+ {
+ currentTransaction = null;
+ }
+ }
+
+ public void rollback()
+ {
+ try
+ {
+ if (currentTransaction != null)
+ {
+ currentTransaction.rollback();
+ }
+ } finally
+ {
+ currentTransaction = null;
}
}
@@ -177,5 +221,4 @@ public class ML2DatabaseUploader implements IDataSetUploader
{
return FilenameUtils.getExtension(incomingDataSetPath.getName()).toLowerCase();
}
-
}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/QuantMLStorageProcessor.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/QuantMLStorageProcessor.java
index 37c5e3c28af531df77ef8e3d2a9401415ae90a57..5f94644be4aa725b2de227616785ec103dad10b6 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/QuantMLStorageProcessor.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/QuantMLStorageProcessor.java
@@ -84,6 +84,27 @@ public class QuantMLStorageProcessor extends AbstractDelegatingStorageProcessor
findFile(incomingDataSetDirectory, mlFileExtension);
}
+ @Override
+ public void commit()
+ {
+ super.commit();
+ databaseUploader.commit();
+ }
+
+ @Override
+ public UnstoreDataAction rollback(final File incomingDataSetDirectory,
+ final File storedDataDirectory, Throwable exception)
+ {
+ try
+ {
+ super.rollback(incomingDataSetDirectory, storedDataDirectory, exception);
+ } finally
+ {
+ databaseUploader.rollback();
+ }
+ return UnstoreDataAction.LEAVE_UNTOUCHED;
+ }
+
// returns the only file with the specified extension or throws an exceptions if none or more
// than one is found.
public static File findFile(File incomingItem, String fileExtension)
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/fiaml/FIAML2Database.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/fiaml/FIAML2Database.java
index fdfffd19bcf79e89b9dfc8ee6165f6a5d3702d3d..af2e0a7b4a48a4ebd56e2f5be194039cca9791c8 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/fiaml/FIAML2Database.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/fiaml/FIAML2Database.java
@@ -23,13 +23,10 @@ import java.util.Iterator;
import javax.sql.DataSource;
import net.lemnik.eodsql.QueryTool;
-import net.lemnik.eodsql.TransactionQuery;
-import org.springframework.dao.DataAccessException;
-
-import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
-import ch.systemsx.cisd.yeastx.db.DBUtils;
+import ch.systemsx.cisd.yeastx.db.AbstractDatasetLoader;
import ch.systemsx.cisd.yeastx.db.DMDataSetDTO;
+import ch.systemsx.cisd.yeastx.db.IGenericDAO;
import ch.systemsx.cisd.yeastx.fiaml.FIAMLParser.IMSRunObserver;
/**
@@ -37,7 +34,7 @@ import ch.systemsx.cisd.yeastx.fiaml.FIAMLParser.IMSRunObserver;
*
* @author Bernd Rinn
*/
-public class FIAML2Database
+public class FIAML2Database extends AbstractDatasetLoader
{
private final static int PROFILE_CHUNK_SIZE = 250;
@@ -87,14 +84,11 @@ public class FIAML2Database
/**
* Method for uploading an <var>fiaMLFile</var> to the database.
*/
- public void uploadFiaMLFile(final File fiaMLFile, final DMDataSetDTO dataSet)
- throws SQLException
+ public void upload(final File fiaMLFile, final DMDataSetDTO dataSet) throws SQLException
{
- TransactionQuery transaction = null;
try
{
- transaction = dao;
- DBUtils.createDataSet(dao, dataSet);
+ createDataSet(dataSet);
new FIAMLParser(fiaMLFile.getPath(), new IMSRunObserver()
{
public void observe(FIAMSRunDTO run, FIAMSRunDataDTO runData)
@@ -108,35 +102,15 @@ public class FIAML2Database
.getCentroidIntensities(), runData.getCentroidCorrelations());
}
});
- transaction.close(true);
} catch (Throwable th)
{
- try
- {
- DBUtils.rollbackAndClose(transaction);
- } catch (DataAccessException ex)
- {
- // Avoid this exception shadowing the original exception.
- }
- throw CheckedExceptionTunnel.wrapIfNecessary(th);
+ rollbackAndRethrow(th);
}
}
- public static void main(String[] args) throws SQLException
+ @Override
+ protected IGenericDAO getDao()
{
- final long start = System.currentTimeMillis();
- final FIAML2Database fiaML2Database =
- new FIAML2Database(DBUtils.createDefaultDBContext().getDataSource());
- final String dir = args[0];
- int permId = 0;
- for (String f : new File(dir).list(new FIAMLFilenameFilter()))
- {
- System.out.println(f);
- fiaML2Database.uploadFiaMLFile(new File(dir, f), new DMDataSetDTO(Integer
- .toString(++permId), "sample perm id", "sample name", "experiment perm id",
- "experiment name"));
- }
- System.out.println((System.currentTimeMillis() - start) / 1000.0);
+ return dao;
}
-
}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/fiaml/FIAMLFilenameFilter.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/fiaml/FIAMLFilenameFilter.java
deleted file mode 100644
index 06e498c8cacfd58138c92d7ff0593920296417c5..0000000000000000000000000000000000000000
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/fiaml/FIAMLFilenameFilter.java
+++ /dev/null
@@ -1,35 +0,0 @@
-/*
- * Copyright 2009 ETH Zuerich, CISD
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package ch.systemsx.cisd.yeastx.fiaml;
-
-import java.io.File;
-import java.io.FilenameFilter;
-
-import ch.systemsx.cisd.yeastx.etl.ConstantsYeastX;
-
-/**
- * A {@link FilenameFilter} for <code>fiaML</code> files.
- *
- * @author Bernd Rinn
- */
-final class FIAMLFilenameFilter implements FilenameFilter
-{
- public boolean accept(File myDir, String name)
- {
- return name.endsWith("." + ConstantsYeastX.FIAML_EXT);
- }
-}
\ No newline at end of file
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/mzxml/MzXml2Database.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/mzxml/MzXml2Database.java
index d2f80f5f0c62e7815264e5c34a86446faaf7e56e..6994ba08de5c28a9613122972cdd157637f66057 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/mzxml/MzXml2Database.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/mzxml/MzXml2Database.java
@@ -24,16 +24,13 @@ import java.util.List;
import javax.sql.DataSource;
import net.lemnik.eodsql.QueryTool;
-import net.lemnik.eodsql.TransactionQuery;
-
-import org.springframework.dao.DataAccessException;
import ch.rinn.restrictions.Private;
-import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.exceptions.NotImplementedException;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
-import ch.systemsx.cisd.yeastx.db.DBUtils;
+import ch.systemsx.cisd.yeastx.db.AbstractDatasetLoader;
import ch.systemsx.cisd.yeastx.db.DMDataSetDTO;
+import ch.systemsx.cisd.yeastx.db.IGenericDAO;
import ch.systemsx.cisd.yeastx.mzxml.dto.MzPrecursorDTO;
import ch.systemsx.cisd.yeastx.mzxml.dto.MzRunDTO;
import ch.systemsx.cisd.yeastx.mzxml.dto.MzScanDTO;
@@ -45,7 +42,7 @@ import ch.systemsx.cisd.yeastx.utils.JaxbXmlParser;
*
* @author Tomasz Pylak
*/
-public class MzXml2Database
+public class MzXml2Database extends AbstractDatasetLoader
{
private final IMzXmlDAO dao;
@@ -63,7 +60,7 @@ public class MzXml2Database
/**
* Method for uploading an <var>mzXML</var> to the database.
*/
- public void uploadFile(final File file, final DMDataSetDTO dataSet) throws SQLException
+ public void upload(final File file, final DMDataSetDTO dataSet) throws SQLException
{
MzXmlDTO mzXml = JaxbXmlParser.parse(MzXmlDTO.class, file, false);
uploadFile(mzXml, dataSet);
@@ -71,23 +68,13 @@ public class MzXml2Database
private void uploadFile(final MzXmlDTO mzXml, final DMDataSetDTO dataSet) throws SQLException
{
- TransactionQuery transaction = null;
try
{
- transaction = dao;
- DBUtils.createDataSet(dao, dataSet);
+ createDataSet(dataSet);
uploadRun(mzXml.getRun(), dataSet);
- transaction.close(true);
} catch (Throwable th)
{
- try
- {
- DBUtils.rollbackAndClose(transaction);
- } catch (DataAccessException ex)
- {
- // Avoid this exception shadowing the original exception.
- }
- throw CheckedExceptionTunnel.wrapIfNecessary(th);
+ rollbackAndRethrow(th);
}
}
@@ -168,4 +155,10 @@ public class MzXml2Database
}
};
}
+
+ @Override
+ protected IGenericDAO getDao()
+ {
+ return dao;
+ }
}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/quant/QuantML2Database.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/quant/QuantML2Database.java
index 0b2713a95e3fcc929bae99a7bc4ce73704962a4b..ef7db3f11e1f5fa86e5c1abc433cc1b43f06f690 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/quant/QuantML2Database.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/quant/QuantML2Database.java
@@ -17,21 +17,16 @@
package ch.systemsx.cisd.yeastx.quant;
import java.io.File;
-import java.io.FilenameFilter;
import java.sql.SQLException;
import java.util.List;
import javax.sql.DataSource;
import net.lemnik.eodsql.QueryTool;
-import net.lemnik.eodsql.TransactionQuery;
-import org.springframework.dao.DataAccessException;
-
-import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
-import ch.systemsx.cisd.yeastx.db.DBUtils;
+import ch.systemsx.cisd.yeastx.db.AbstractDatasetLoader;
import ch.systemsx.cisd.yeastx.db.DMDataSetDTO;
-import ch.systemsx.cisd.yeastx.etl.ConstantsYeastX;
+import ch.systemsx.cisd.yeastx.db.IGenericDAO;
import ch.systemsx.cisd.yeastx.quant.dto.ConcentrationCompounds;
import ch.systemsx.cisd.yeastx.quant.dto.MSConcentrationDTO;
import ch.systemsx.cisd.yeastx.quant.dto.MSQuantificationDTO;
@@ -43,9 +38,8 @@ import ch.systemsx.cisd.yeastx.utils.JaxbXmlParser;
*
* @author Tomasz Pylak
*/
-public class QuantML2Database
+public class QuantML2Database extends AbstractDatasetLoader
{
-
private final IQuantMSDAO dao;
public QuantML2Database(DataSource datasource)
@@ -56,27 +50,17 @@ public class QuantML2Database
/**
* Method for uploading an <var>fiaMLFile</var> to the database.
*/
- public void uploadQuantMLFile(final File file, final DMDataSetDTO dataSet) throws SQLException
+ public void upload(final File file, final DMDataSetDTO dataSet) throws SQLException
{
- TransactionQuery transaction = null;
try
{
- transaction = dao;
- DBUtils.createDataSet(dao, dataSet);
+ createDataSet(dataSet);
MSQuantificationsDTO quantifications =
JaxbXmlParser.parse(MSQuantificationsDTO.class, file, false);
uploadQuantifications(quantifications, dataSet);
- transaction.close(true);
} catch (Throwable th)
{
- try
- {
- DBUtils.rollbackAndClose(transaction);
- } catch (DataAccessException ex)
- {
- // Avoid this exception shadowing the original exception.
- }
- throw CheckedExceptionTunnel.wrapIfNecessary(th);
+ rollbackAndRethrow(th);
}
}
@@ -105,32 +89,10 @@ public class QuantML2Database
dao.addCompoundIds(concentrationId, compounds.getCompoundIds());
}
- public static void main(String[] args) throws SQLException
- {
- final long start = System.currentTimeMillis();
- final QuantML2Database quantML2Database =
- new QuantML2Database(DBUtils.createDefaultDBContext().getDataSource());
- final String dir = args[0];
- int permId = 0;
- for (String f : new File(dir).list(createQuantFilter()))
- {
- System.out.println(f);
- quantML2Database.uploadQuantMLFile(new File(dir, f), new DMDataSetDTO(Integer
- .toString(++permId), "sample perm id", "sample name", "experiment perm id",
- "experiment name"));
- }
- System.out.println((System.currentTimeMillis() - start) / 1000.0);
- }
-
- private static FilenameFilter createQuantFilter()
+ @Override
+ protected IGenericDAO getDao()
{
- return new FilenameFilter()
- {
- public boolean accept(File dir, String name)
- {
- return name.endsWith("." + ConstantsYeastX.QUANTML_EXT);
- }
- };
+ return dao;
}
}
diff --git a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/eicml/EICMLTest.java b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/eicml/EICMLTest.java
index f8d194dd8a8bd96e3d62056eb2159fbebb355711..6616901c68501f577db36c23518c61103b161c6d 100644
--- a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/eicml/EICMLTest.java
+++ b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/eicml/EICMLTest.java
@@ -53,9 +53,15 @@ public class EICMLTest extends AbstractDBTest
public void testUploadEicML() throws SQLException
{
EICML2Database db = new EICML2Database(datasource);
- db.uploadEicMLFile(new File("resource/examples/example.eicML"), new DMDataSetDTO(
- "data set perm id eicml", "sample perm id eicml", "sample name eicml",
- "experiment perm id eicml", "experiment name eicml"));
+ // TODO 2009-11-11, Tomasz Pylak: write rollback test
+ db.upload(new File("resource/examples/example.eicML"), createDataset());
+ db.commit();
+ }
+
+ private DMDataSetDTO createDataset()
+ {
+ return new DMDataSetDTO("data set perm id eicml", "sample perm id eicml",
+ "sample name eicml", "experiment perm id eicml", "experiment name eicml");
}
private void checkChromatograms(int count, ChromatogramDTO chrom)
diff --git a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/fiaml/FIAMLTest.java b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/fiaml/FIAMLTest.java
index 4cd69cacbe501b92f86c4830ed2258fa75b13f91..1edf03db09dd78b828b457f21512e9b03508b812 100644
--- a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/fiaml/FIAMLTest.java
+++ b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/fiaml/FIAMLTest.java
@@ -59,9 +59,10 @@ public class FIAMLTest extends AbstractDBTest
public void testUploadFiaML() throws SQLException
{
FIAML2Database fiaML2Database = new FIAML2Database(datasource);
- fiaML2Database.uploadFiaMLFile(new File("resource/examples/example.fiaML"),
- new DMDataSetDTO("data set perm id fiaml", "sample perm id", "sample name",
- "experiment perm id", "experiment name"));
+ fiaML2Database.upload(new File("resource/examples/example.fiaML"), new DMDataSetDTO(
+ "data set perm id fiaml", "sample perm id", "sample name", "experiment perm id",
+ "experiment name"));
+ fiaML2Database.commit();
}
@Test(dependsOnMethods = "testUploadFiaML")
@@ -99,12 +100,14 @@ public class FIAMLTest extends AbstractDBTest
public void testUploadFiaMLFileTwiceWithOneDAO() throws SQLException
{
FIAML2Database fiaML2Database = new FIAML2Database(datasource);
- fiaML2Database.uploadFiaMLFile(new File("resource/examples/example.fiaML"),
- new DMDataSetDTO("data set perm id2", "sample perm id2", "sample name",
- "experiment perm id2", "experiment name"));
- fiaML2Database.uploadFiaMLFile(new File("resource/examples/example.fiaML"),
- new DMDataSetDTO("data set perm id3", "sample perm id", "sample name",
- "experiment perm id", "experiment name"));
+ fiaML2Database.upload(new File("resource/examples/example.fiaML"), new DMDataSetDTO(
+ "data set perm id2", "sample perm id2", "sample name", "experiment perm id2",
+ "experiment name"));
+ fiaML2Database.commit();
+ fiaML2Database.upload(new File("resource/examples/example.fiaML"), new DMDataSetDTO(
+ "data set perm id3", "sample perm id", "sample name", "experiment perm id",
+ "experiment name"));
+ fiaML2Database.commit();
}
@Test(dependsOnMethods = "testUploadFiaMLFileTwiceWithOneDAO", expectedExceptions = DataIntegrityViolationException.class)
diff --git a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/mzxml/MzXmlDAOTest.java b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/mzxml/MzXmlDAOTest.java
index e80a6378b288c34df9dbc2aa855c53451bf8103d..c4c5b49ee4ceec83f1d9b686335148c3e30ac0c7 100644
--- a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/mzxml/MzXmlDAOTest.java
+++ b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/mzxml/MzXmlDAOTest.java
@@ -57,11 +57,14 @@ public class MzXmlDAOTest extends AbstractDBTest
DMDataSetDTO dataSetDTO =
new DMDataSetDTO("data set perm id mzXML", "sample perm id", "sample name",
"experiment perm id", "experiment name");
- uploader.uploadFile(new File("resource/examples/example.mzXML"), dataSetDTO);
+ uploader.upload(new File("resource/examples/example.mzXML"), dataSetDTO);
+ uploader.commit();
dataSetDTO.setPermId("data set perm id mzXML minimal");
- uploader.uploadFile(new File("resource/examples/example-fake-minimal.mzXML"), dataSetDTO);
+ uploader.upload(new File("resource/examples/example-fake-minimal.mzXML"), dataSetDTO);
+ uploader.commit();
dataSetDTO.setPermId("data set perm id mzXML full");
- uploader.uploadFile(new File("resource/examples/example-fake-full.mzXML"), dataSetDTO);
+ uploader.upload(new File("resource/examples/example-fake-full.mzXML"), dataSetDTO);
+ uploader.commit();
}
}
diff --git a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/quant/QuantMSDAOTest.java b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/quant/QuantMSDAOTest.java
index dc5e9a52bd173e784cc9fc799da7c218be814fb0..95de9c5edb850b65bf7af97aecc687c86d73fe64 100644
--- a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/quant/QuantMSDAOTest.java
+++ b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/quant/QuantMSDAOTest.java
@@ -57,7 +57,7 @@ public class QuantMSDAOTest extends AbstractDBTest
DMDataSetDTO dataSetDTO =
new DMDataSetDTO("data set perm id quant", "sample perm id quant", "sample name",
"experiment perm id", "experiment name");
- uploader.uploadQuantMLFile(new File("resource/examples/allFields.quantML"), dataSetDTO);
+ uploader.upload(new File("resource/examples/allFields.quantML"), dataSetDTO);
}
}