From a6b8eb2a4b911e1ae9f10261d87cd28dcbc9e50f Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Wed, 21 Jul 2010 11:29:47 +0000
Subject: [PATCH] LMS-1652 screening migration: fixes
SVN: 17115
---
.../cisd/dbmigration/java/IMigrationStep.java | 10 ++-
.../java/MigrationStepAdapter.java | 10 ++-
.../java/MigrationStepExecutor.java | 14 +--
.../java/MigrationStepAdminFrom001To002.java | 14 +--
.../java/MigrationStepFrom001To002.java | 6 +-
.../java/MigrationStepFrom002To003.java | 12 +--
.../migration/MigrationStepFrom022To023.java | 10 ++-
.../migration/MigrationStepFrom023To024.java | 9 +-
.../migration/MigrationStepFrom025To026.java | 6 +-
.../migration/MigrationStepFrom036To037.java | 6 +-
.../migration/MigrationStepFrom046To047.java | 6 +-
.../migration/MigrationStepFrom002To003.java | 53 +++++------
.../migration/MigrationStepFrom004To005.java | 7 +-
.../dss/etl/ImagingDatabaseVersionHolder.java | 4 +-
.../migration/MigrationStepFrom003To004.java | 89 ++++++++++++-------
.../featurevector/FeatureVectorUploader.java | 7 +-
16 files changed, 153 insertions(+), 110 deletions(-)
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/IMigrationStep.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/IMigrationStep.java
index 7b9aa7389d0..34f3026f317 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/IMigrationStep.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/IMigrationStep.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.dbmigration.java;
+import javax.sql.DataSource;
+
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
@@ -43,12 +45,12 @@ public interface IMigrationStep
/**
* Called before the SQL migration is performed.
*/
- public void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException;
+ public void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException;
/**
* Called after the SQL migration has been performed.
*/
- public void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException;
+ public void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException;
}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/MigrationStepAdapter.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/MigrationStepAdapter.java
index 408f4a1f2ac..599dcbab6e8 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/MigrationStepAdapter.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/MigrationStepAdapter.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.dbmigration.java;
+import javax.sql.DataSource;
+
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
@@ -34,13 +36,13 @@ public class MigrationStepAdapter implements IMigrationStep
// IMigrationStep
//
- public void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
}
- public void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/MigrationStepExecutor.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/MigrationStepExecutor.java
index e4e601be24e..c86915657ca 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/MigrationStepExecutor.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/java/MigrationStepExecutor.java
@@ -78,12 +78,12 @@ public class MigrationStepExecutor extends SimpleJdbcDaoSupport implements IMigr
}
final ParserUtilities.LineSplitter splitter =
new ParserUtilities.LineSplitter(content, new ILineFilter()
- {
- public boolean acceptLine(String line, int lineNumber)
{
- return StringUtils.isNotBlank(line) && line.startsWith("--");
- }
- });
+ public boolean acceptLine(String line, int lineNumber)
+ {
+ return StringUtils.isNotBlank(line) && line.startsWith("--");
+ }
+ });
IMigrationStep stepOrNull = null;
Line lineOrNull;
while (stepOrNull == null && (lineOrNull = splitter.tryNextLine()) != null)
@@ -151,7 +151,7 @@ public class MigrationStepExecutor extends SimpleJdbcDaoSupport implements IMigr
assert inited : "Executor not initialized.";
if (migrationStep != null)
{
- migrationStep.performPreMigration(getSimpleJdbcTemplate());
+ migrationStep.performPreMigration(getSimpleJdbcTemplate(), getDataSource());
}
}
@@ -160,7 +160,7 @@ public class MigrationStepExecutor extends SimpleJdbcDaoSupport implements IMigr
assert inited : "Executor not initialized.";
if (migrationStep != null)
{
- migrationStep.performPostMigration(getSimpleJdbcTemplate());
+ migrationStep.performPostMigration(getSimpleJdbcTemplate(), getDataSource());
}
}
diff --git a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepAdminFrom001To002.java b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepAdminFrom001To002.java
index 6e0bc139ed3..729ca81eda8 100644
--- a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepAdminFrom001To002.java
+++ b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepAdminFrom001To002.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.dbmigration.java;
+import javax.sql.DataSource;
+
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
import org.testng.AssertJUnit;
@@ -33,9 +35,9 @@ public final class MigrationStepAdminFrom001To002 implements IMigrationStep
public static MigrationStepAdminFrom001To002 instance;
public final DatabaseConfigurationContext context;
-
+
public boolean preMigrationPerformed;
-
+
public boolean postMigrationPerformed;
public MigrationStepAdminFrom001To002(DatabaseConfigurationContext context)
@@ -50,14 +52,14 @@ public final class MigrationStepAdminFrom001To002 implements IMigrationStep
// IMigrationStep
//
- public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
postMigrationPerformed = true;
}
- public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
preMigrationPerformed = true;
}
diff --git a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepFrom001To002.java b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepFrom001To002.java
index f82fa54a83c..439c6a2a5a5 100644
--- a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepFrom001To002.java
+++ b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepFrom001To002.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.dbmigration.java;
+import javax.sql.DataSource;
+
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
@@ -31,12 +33,12 @@ public final class MigrationStepFrom001To002 implements IMigrationStep
// IMigrationStep
//
- public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
+ public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate, DataSource dataSource)
throws DataAccessException
{
}
- public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
+ public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate, DataSource dataSource)
throws DataAccessException
{
}
diff --git a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepFrom002To003.java b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepFrom002To003.java
index 640d3eb50b2..9b0b8c47d80 100644
--- a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepFrom002To003.java
+++ b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/java/MigrationStepFrom002To003.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.dbmigration.java;
+import javax.sql.DataSource;
+
import org.apache.commons.lang.StringUtils;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.DataIntegrityViolationException;
@@ -36,19 +38,19 @@ public final class MigrationStepFrom002To003 implements IMigrationStep
{
AssertJUnit.assertNotNull(context);
}
-
+
//
// IMigrationStep
//
- public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
throw new EmptyResultDataAccessException(1);
}
- public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
throw new DataIntegrityViolationException(StringUtils.EMPTY);
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom022To023.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom022To023.java
index 98ca2991d31..f1341919896 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom022To023.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom022To023.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.openbis.generic.server.dataaccess.migration;
+import javax.sql.DataSource;
+
import org.springframework.dao.DataAccessException;
import org.springframework.dao.IncorrectResultSizeDataAccessException;
import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
@@ -40,8 +42,8 @@ public final class MigrationStepFrom022To023 implements IMigrationStep
// IMigrationStep
//
- public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
final String uuid = UuidUtil.generateUUID();
final int count =
@@ -57,8 +59,8 @@ public final class MigrationStepFrom022To023 implements IMigrationStep
}
}
- public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
}
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom023To024.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom023To024.java
index 4da1aaee7f8..ee59d53ad29 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom023To024.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom023To024.java
@@ -38,8 +38,9 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.TableNames;
/**
* A migration step from database version <code>v23</code> to version <code>v24</code>.
* <p>
- * This migration step does the following: it migrate the dataset locations in the database (<code>Instance_<instance code></code>
- * renamed Instance_<UUID> when <instance code> is the original source).
+ * This migration step does the following: it migrate the dataset locations in the database (
+ * <code>Instance_<instance code></code> renamed Instance_<UUID> when <instance
+ * code> is the original source).
* </p>
*
* @author Christian Ribeaud
@@ -112,8 +113,8 @@ public final class MigrationStepFrom023To024 extends MigrationStepAdapter
//
@Override
- public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ javax.sql.DataSource dataSource) throws DataAccessException
{
final DatabaseInstance databaseInstance = getDatabaseInstance(simpleJdbcTemplate);
final List<ExternalData> externalDatas =
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom025To026.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom025To026.java
index 74d5a39e122..074f0b3189e 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom025To026.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom025To026.java
@@ -21,6 +21,8 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
+import javax.sql.DataSource;
+
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.log4j.Logger;
import org.springframework.dao.DataAccessException;
@@ -89,8 +91,8 @@ public final class MigrationStepFrom025To026 extends MigrationStepAdapter
//
@Override
- public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
final List<ExternalData> externalDatas =
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom036To037.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom036To037.java
index f59cbda3101..d5a848e7a91 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom036To037.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom036To037.java
@@ -21,6 +21,8 @@ import java.sql.SQLException;
import java.util.ArrayList;
import java.util.List;
+import javax.sql.DataSource;
+
import org.apache.commons.lang.builder.ToStringBuilder;
import org.apache.log4j.Logger;
import org.springframework.dao.DataAccessException;
@@ -79,8 +81,8 @@ public final class MigrationStepFrom036To037 extends MigrationStepAdapter
//
@Override
- public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPostMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
for (EntityKind entityKind : EntityKind.values())
{
diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom046To047.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom046To047.java
index 6858e3d8cdf..9328916592f 100644
--- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom046To047.java
+++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/migration/MigrationStepFrom046To047.java
@@ -20,6 +20,8 @@ import java.sql.ResultSet;
import java.sql.SQLException;
import java.util.List;
+import javax.sql.DataSource;
+
import org.apache.log4j.Logger;
import org.springframework.dao.DataAccessException;
import org.springframework.dao.IncorrectResultSizeDataAccessException;
@@ -77,8 +79,8 @@ public final class MigrationStepFrom046To047 extends MigrationStepAdapter
//
@Override
- public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate)
- throws DataAccessException
+ public final void performPreMigration(final SimpleJdbcTemplate simpleJdbcTemplate,
+ DataSource dataSource) throws DataAccessException
{
final List<PersonWithDisplaySettings> persons =
simpleJdbcTemplate.query(String.format(SELECT_PERSON_WITH_DISPLAY_SETTINGS_QUERY,
diff --git a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/plugin/phosphonetx/server/dataaccess/db/migration/MigrationStepFrom002To003.java b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/plugin/phosphonetx/server/dataaccess/db/migration/MigrationStepFrom002To003.java
index ec250df1430..fdba93930a8 100644
--- a/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/plugin/phosphonetx/server/dataaccess/db/migration/MigrationStepFrom002To003.java
+++ b/rtd_phosphonetx/source/java/ch/systemsx/cisd/openbis/plugin/phosphonetx/server/dataaccess/db/migration/MigrationStepFrom002To003.java
@@ -25,6 +25,8 @@ import java.util.List;
import java.util.Map;
import java.util.Set;
+import javax.sql.DataSource;
+
import org.apache.log4j.Logger;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.JdbcOperations;
@@ -38,25 +40,25 @@ import ch.systemsx.cisd.openbis.plugin.phosphonetx.shared.basic.dto.Occurrence;
import ch.systemsx.cisd.openbis.plugin.phosphonetx.shared.basic.dto.OccurrenceUtil;
/**
- *
- *
* @author Franz-Josef Elmer
*/
public class MigrationStepFrom002To003 extends MigrationStepAdapter
{
private static final long MB = 1024 * 1024;
+
private static final Logger operationLog =
- LogFactory.getLogger(LogCategory.OPERATION, MigrationStepFrom002To003.class);
-
+ LogFactory.getLogger(LogCategory.OPERATION, MigrationStepFrom002To003.class);
+
@Override
- public void performPostMigration(SimpleJdbcTemplate simpleJdbcTemplate)
+ public void performPostMigration(SimpleJdbcTemplate simpleJdbcTemplate, DataSource dataSource)
throws DataAccessException
{
List<Object[]> coverageValues = calculateCoverageValues(simpleJdbcTemplate);
operationLog.info("update " + coverageValues.size() + " identified proteins");
- simpleJdbcTemplate.batchUpdate("update identified_proteins set coverage = ? where id = ?", coverageValues);
+ simpleJdbcTemplate.batchUpdate("update identified_proteins set coverage = ? where id = ?",
+ coverageValues);
}
-
+
private List<Object[]> calculateCoverageValues(SimpleJdbcTemplate simpleJdbcTemplate)
{
logMemory();
@@ -64,31 +66,31 @@ public class MigrationStepFrom002To003 extends MigrationStepAdapter
final Map<Long, List<String>> peptides = getPeptides(jdbcOperations);
logMemory();
final List<Object[]> values = new ArrayList<Object[]>();
- jdbcOperations.query(
- "select ip.id, ip.prot_id, s.amino_acid_sequence " +
- "from identified_proteins as ip " +
- "join sequences as s on ip.sequ_id = s.id where coverage is null",
+ jdbcOperations.query("select ip.id, ip.prot_id, s.amino_acid_sequence "
+ + "from identified_proteins as ip "
+ + "join sequences as s on ip.sequ_id = s.id where coverage is null",
new ResultSetExtractor()
- {
- public Object extractData(ResultSet rs) throws SQLException,
- DataAccessException
{
- while (rs.next())
+ public Object extractData(ResultSet rs) throws SQLException,
+ DataAccessException
{
- long id = rs.getLong(1);
- long proteinID = rs.getLong(2);
- String sequence = rs.getString(3);
- List<String> peptideSequences = peptides.get(proteinID);
- double coverage = calculateCoverage(sequence, peptideSequences);
- values.add(new Object[] { coverage, id });
+ while (rs.next())
+ {
+ long id = rs.getLong(1);
+ long proteinID = rs.getLong(2);
+ String sequence = rs.getString(3);
+ List<String> peptideSequences = peptides.get(proteinID);
+ double coverage = calculateCoverage(sequence, peptideSequences);
+ values.add(new Object[]
+ { coverage, id });
+ }
+ return null;
}
- return null;
- }
- });
+ });
logMemory();
return values;
}
-
+
private Map<Long, List<String>> getPeptides(JdbcOperations jdbcOperations)
{
final Map<Long, List<String>> peptides = new HashMap<Long, List<String>>();
@@ -140,4 +142,3 @@ public class MigrationStepFrom002To003 extends MigrationStepAdapter
operationLog.info(usedMemory + " MB used, " + runtime.totalMemory() / MB + " MB total");
}
}
-
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/migration/MigrationStepFrom004To005.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/migration/MigrationStepFrom004To005.java
index b8485a92861..8fadc0473a9 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/migration/MigrationStepFrom004To005.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/db/migration/MigrationStepFrom004To005.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.yeastx.db.migration;
+import javax.sql.DataSource;
+
import org.apache.log4j.Logger;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.simple.SimpleJdbcTemplate;
@@ -25,8 +27,6 @@ import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.dbmigration.java.MigrationStepAdapter;
/**
- *
- *
* @author Franz-Josef Elmer
*/
public class MigrationStepFrom004To005 extends MigrationStepAdapter
@@ -35,7 +35,7 @@ public class MigrationStepFrom004To005 extends MigrationStepAdapter
LogFactory.getLogger(LogCategory.OPERATION, MigrationStepFrom004To005.class);
@Override
- public void performPostMigration(SimpleJdbcTemplate simpleJdbcTemplate)
+ public void performPostMigration(SimpleJdbcTemplate simpleJdbcTemplate, DataSource dataSource)
throws DataAccessException
{
operationLog.warn("!!!!!!!! METABOL DATABASE MIGRATED WITH OUT MIGRATING TABLE MZ_PEAKS. "
@@ -43,5 +43,4 @@ public class MigrationStepFrom004To005 extends MigrationStepAdapter
+ "Please, delete MZ_PEAKS manually if it no longer needed.");
}
-
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseVersionHolder.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseVersionHolder.java
index 9bf19aec9c0..16f79dcb957 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseVersionHolder.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseVersionHolder.java
@@ -20,7 +20,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.IDatabaseVersionHolder;
/**
* Version holder for imaging database.
- *
+ *
* @author Franz-Josef Elmer
*/
public class ImagingDatabaseVersionHolder implements IDatabaseVersionHolder
@@ -28,7 +28,7 @@ public class ImagingDatabaseVersionHolder implements IDatabaseVersionHolder
public String getDatabaseVersion()
{
- return "003"; // S83
+ return "004"; // changed in S86
}
}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/migration/MigrationStepFrom003To004.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/migration/MigrationStepFrom003To004.java
index 32244776695..9cbfd37fa08 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/migration/MigrationStepFrom003To004.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dataaccess/migration/MigrationStepFrom003To004.java
@@ -25,6 +25,10 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import javax.sql.DataSource;
+
+import net.lemnik.eodsql.QueryTool;
+
import org.apache.log4j.Logger;
import org.springframework.dao.DataAccessException;
import org.springframework.jdbc.core.simple.ParameterizedRowMapper;
@@ -44,7 +48,6 @@ import ch.systemsx.cisd.openbis.dss.etl.featurevector.CsvToCanonicalFeatureVecto
import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.DatasetFileLines;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DatasetLocationUtil;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUtil;
-import ch.systemsx.cisd.openbis.dss.shared.DssScreeningUtils;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ScreeningConstants;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.utils.CsvFileReaderHelper;
@@ -58,8 +61,6 @@ import ch.systemsx.cisd.utils.CsvFileReaderHelper.ICsvFileReaderConfiguration;
*/
public class MigrationStepFrom003To004 implements IMigrationStep
{
- private static final char DEFAULT_COLUMNS_SEPARATOR = ',';
-
private static final Logger operationLog =
LogFactory.getLogger(LogCategory.OPERATION, MigrationStepFrom003To004.class);
@@ -119,8 +120,10 @@ public class MigrationStepFrom003To004 implements IMigrationStep
}
}
- public void performPostMigration(SimpleJdbcTemplate jdbc) throws DataAccessException
+ public void performPostMigration(SimpleJdbcTemplate jdbc, DataSource dataSource)
+ throws DataAccessException
{
+ IImagingQueryDAO dao = QueryTool.getQuery(dataSource, IImagingQueryDAO.class);
File storeRootDir = getStoreRootDir();
String dbUUID = tryGetDatabaseInstanceUUID(storeRootDir);
if (dbUUID == null)
@@ -131,52 +134,58 @@ public class MigrationStepFrom003To004 implements IMigrationStep
List<MigrationDatasetRef> datasets = fetchImagingDatasets(jdbc);
Map<MigrationDatasetRef, DatasetFileLines> fileMap =
createFileMap(datasets, storeRootDir, dbUUID);
- boolean ok = migrateDatasets(fileMap, jdbc);
+ boolean ok = migrateDatasets(fileMap, jdbc, dao);
+ dao.commit();
+ dao.close();
if (ok == false)
{
- throw new EnvironmentFailureException("Feature vector migration failed!");
+ operationLog.warn("There were some error during feature vector migration!");
}
}
private boolean migrateDatasets(Map<MigrationDatasetRef, DatasetFileLines> fileMap,
- SimpleJdbcTemplate jdbc)
+ SimpleJdbcTemplate jdbc, IImagingQueryDAO dao)
{
boolean wholeMigrationOk = true;
for (Entry<MigrationDatasetRef, DatasetFileLines> entry : fileMap.entrySet())
{
long datasetId = entry.getKey().getId();
+ String permId = entry.getKey().getPermId();
DatasetFileLines featureVectorLines = entry.getValue();
try
{
- migrateDataset(jdbc, datasetId, featureVectorLines);
+ operationLog.info("Migrating dataset: " + permId);
+ migrateDataset(jdbc, dao, datasetId, featureVectorLines);
} catch (Exception ex)
{
- operationLog.error("Cannot migrate dataset " + entry.getKey().getPermId() + ": "
- + ex.getMessage());
+ operationLog.error("Cannot migrate dataset " + permId + ": " + ex.getMessage());
+ if (ex instanceof IllegalArgumentException == false)
+ {
+ ex.printStackTrace();
+ }
wholeMigrationOk = false;
}
}
return wholeMigrationOk;
}
- private void migrateDataset(SimpleJdbcTemplate jdbc, long datasetId,
+ private void migrateDataset(SimpleJdbcTemplate jdbc, IImagingQueryDAO dao, long datasetId,
DatasetFileLines featureVectorLines)
{
List<CanonicalFeatureVector> fvecs = extractFeatureVectors(featureVectorLines);
- deleteFeatureVectors(datasetId, jdbc);
- uploadFeatureVectors(datasetId, fvecs);
- }
-
- private void deleteFeatureVectors(long datasetId, SimpleJdbcTemplate jdbc)
- {
- int deleted = jdbc.update("delete from feature_defs defs where defs.ds_id = ?", datasetId);
- operationLog.info(String.format("%d features deleted for the dataset %d.", deleted,
- datasetId));
+ int deleted = deleteFeatureVectors(datasetId, jdbc);
+ if (deleted != fvecs.size())
+ {
+ operationLog.error(String.format(
+ "Dataset techId(%d) had %d features, but now it has %d.", datasetId, deleted,
+ fvecs.size()));
+ }
+ uploadFeatureVectors(datasetId, fvecs, dao);
}
- private void uploadFeatureVectors(long datasetId, List<CanonicalFeatureVector> fvecs)
+ private void uploadFeatureVectors(long datasetId, List<CanonicalFeatureVector> fvecs,
+ IImagingQueryDAO dao)
{
- IImagingQueryDAO dao = DssScreeningUtils.createQuery();
FeatureVectorUploader.uploadFeatureVectors(dao, fvecs, datasetId);
}
@@ -187,24 +196,31 @@ public class MigrationStepFrom003To004 implements IMigrationStep
return new CsvToCanonicalFeatureVector(featureVectorLines, convertorConfig).convert();
}
- private static DatasetFileLines getDatasetFileLines(File file) throws IOException
+ private static DatasetFileLines getDatasetFileLines(File file, final char separator)
+ throws IOException
{
ICsvFileReaderConfiguration configuration = new DefaultCsvFileReaderConfiguration()
{
@Override
public char getColumnDelimiter()
{
- return DEFAULT_COLUMNS_SEPARATOR;
+ return separator;
}
};
return CsvFileReaderHelper.getDatasetFileLines(file, configuration);
}
+ private int deleteFeatureVectors(long datasetId, SimpleJdbcTemplate jdbc)
+ {
+ return jdbc.update("delete from feature_defs defs where defs.ds_id = ?", datasetId);
+ }
+
private List<MigrationDatasetRef> fetchImagingDatasets(SimpleJdbcTemplate simpleJdbcTemplate)
{
- return simpleJdbcTemplate.query(
- "select id, perm_id from feature_defs defs, data_sets d where d.if = defs.ds_id",
- DATASET_ROW_MAPPER);
+ return simpleJdbcTemplate
+ .query(
+ "select distinct d.id, d.perm_id from feature_defs defs, data_sets d where d.id = defs.ds_id",
+ DATASET_ROW_MAPPER);
}
private String tryGetDatabaseInstanceUUID(File storeRootDir)
@@ -244,13 +260,18 @@ public class MigrationStepFrom003To004 implements IMigrationStep
File[] datasetFiles = origDir.listFiles();
if (datasetFiles == null || datasetFiles.length == 0)
{
+ operationLog.warn("Empty dataset dir: " + datasetDir);
return null;
}
for (File datasetFile : datasetFiles)
{
- DatasetFileLines fileLines = tryReadFeatureVectors(datasetFile);
- if (fileLines != null)
+ DatasetFileLines fileLines = tryReadFeatureVectors(datasetFile, ',');
+ if (fileLines == null || fileLines.getHeaderTokens().length <= 2)
+ {
+ fileLines = tryReadFeatureVectors(datasetFile, ';');
+ }
+ if (fileLines != null && fileLines.getHeaderTokens().length > 2)
{
return fileLines;
}
@@ -258,15 +279,15 @@ public class MigrationStepFrom003To004 implements IMigrationStep
throw new EnvironmentFailureException(
"Cannot find the file with feature vectors for the dataset. "
+ "Delete this dataset from openBIS and restart the server to perform migration again. Dataset: "
- + datasetDir.getName() + ". Diretcory: " + datasetDir);
+ + datasetDir.getName() + ". Directory: " + datasetDir);
}
- private static DatasetFileLines tryReadFeatureVectors(File datasetFile)
+ private static DatasetFileLines tryReadFeatureVectors(File datasetFile, char separator)
{
try
{
- return getDatasetFileLines(datasetFile);
- } catch (IOException ex)
+ return getDatasetFileLines(datasetFile, separator);
+ } catch (Exception ex)
{
operationLog.warn("Cannot read the file or file has the wrong format: " + datasetFile
+ ": " + ex.getMessage());
@@ -274,7 +295,7 @@ public class MigrationStepFrom003To004 implements IMigrationStep
}
}
- public void performPreMigration(SimpleJdbcTemplate simpleJdbcTemplate)
+ public void performPreMigration(SimpleJdbcTemplate simpleJdbcTemplate, DataSource dataSource)
throws DataAccessException
{
// do nothing
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploader.java
index bd1bfa7df6b..93a7ac8d55f 100644
--- a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploader.java
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/featurevector/FeatureVectorUploader.java
@@ -41,7 +41,10 @@ public class FeatureVectorUploader
this.info = info;
}
- /** creates a dataset and uploads feature vectors, creates experiment and container if needed */
+ /**
+ * Creates a dataset and uploads feature vectors, creates experiment and container if needed.
+ * Commit on the dao is NOT performed.
+ */
public void uploadFeatureVectors(List<CanonicalFeatureVector> fvecs)
{
ScreeningContainerDatasetInfoHelper helper = new ScreeningContainerDatasetInfoHelper(dao);
@@ -51,7 +54,7 @@ public class FeatureVectorUploader
uploadFeatureVectors(dao, fvecs, dataSetId);
}
- /** uploads feature vectors for a given dataset id */
+ /** Uploads feature vectors for a given dataset id. Commit on the dao is NOT performed. */
public static void uploadFeatureVectors(IImagingQueryDAO dao,
List<CanonicalFeatureVector> fvecs, long dataSetId)
{
--
GitLab