From 98a091a5577be4bf09450ec788c999a35f55aef7 Mon Sep 17 00:00:00 2001
From: brinn <brinn>
Date: Sun, 6 Jan 2008 13:46:11 +0000
Subject: [PATCH] refactor: make database initialization simpler (dependent on
less variables) and easier to generalize to other db engines change: - now
every directory containing a database dump needs to be tagged with a .DUMP
file - add the concept of "generic" schema scripts
SVN: 3311
---
.../cisd/dbmigration/DBMigrationEngine.java | 29 +---
.../DatabaseConfigurationContext.java | 52 ------
.../cisd/dbmigration/IDatabaseAdminDAO.java | 6 +-
.../cisd/dbmigration/ISqlScriptProvider.java | 27 +--
.../cisd/dbmigration/SqlScriptProvider.java | 146 +++++-----------
.../postgresql/PostgreSQLAdminDAO.java | 117 ++++++++++---
.../cisd/dbmigration/postgresql/createLog.sql | 8 -
.../dbmigration/DBMigrationEngineTest.java | 118 +------------
.../dbmigration/SqlScriptProviderTest.java | 164 ++++++++----------
9 files changed, 236 insertions(+), 431 deletions(-)
delete mode 100644 dbmigration/source/java/ch/systemsx/cisd/dbmigration/postgresql/createLog.sql
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DBMigrationEngine.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DBMigrationEngine.java
index 9ba6174ca79..f8ebca7b3fe 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DBMigrationEngine.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DBMigrationEngine.java
@@ -20,10 +20,10 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.Script;
import ch.systemsx.cisd.common.db.ISqlScriptExecutor;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.common.utilities.OSUtilities;
/**
* Class for creating and migrating a database.
@@ -61,8 +61,8 @@ public class DBMigrationEngine
/**
* Create or migrate database to the specified version.
*
- * @throws EnvironmentFailureException if creation/migration failed because of some missing scripts or an
- * inconsistent database.
+ * @throws ConfigurationFailureException If creation/migration fails due to a missing script
+ * @throws EnvironmentFailureException If creation/migration fails due to an inconsistent database.
*/
public void migrateTo(String version)
{
@@ -133,7 +133,7 @@ public class DBMigrationEngine
adminDAO.createOwner();
if (scriptProvider.isDumpRestore(version))
{
- adminDAO.restoreDatabaseFromDump(scriptProvider, version);
+ adminDAO.restoreDatabaseFromDump(scriptProvider.getDumpFolder(version), version);
} else
{
createEmptyDatabase(version);
@@ -148,24 +148,7 @@ public class DBMigrationEngine
private void createEmptyDatabase(String version)
{
- adminDAO.createDatabase();
- Script createScript = scriptProvider.tryGetLogCreationScript();
- if (createScript == null)
- {
- String message = "Missing log creation script";
- operationLog.error(message);
- throw new EnvironmentFailureException(message);
- }
- try
- {
- logDAO.createTable(createScript);
- } catch (RuntimeException e)
- {
- operationLog.error("Script '" + createScript.getName() + "' failed:" + OSUtilities.LINE_SEPARATOR
- + createScript.getCode(), e);
- throw e;
- }
-
+ adminDAO.createDatabase();
executeSchemaScript(version);
}
@@ -176,7 +159,7 @@ public class DBMigrationEngine
{
final String message = "No schema script found for version " + version;
operationLog.error(message);
- throw new EnvironmentFailureException(message);
+ throw new ConfigurationFailureException(message);
}
scriptExecutor.execute(schemaScript, true, logDAO);
final Script functionScript = scriptProvider.tryGetFunctionScript(version);
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DatabaseConfigurationContext.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DatabaseConfigurationContext.java
index f91c1cf3149..30a1ebac52c 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DatabaseConfigurationContext.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DatabaseConfigurationContext.java
@@ -57,10 +57,6 @@ public class DatabaseConfigurationContext implements DisposableBean
private String scriptFolder;
- private String folderOfDataScripts;
-
- private String folderOfMassUploadFiles;
-
private String databaseKind;
private DatabaseEngine databaseEngine;
@@ -501,54 +497,6 @@ public class DatabaseConfigurationContext implements DisposableBean
this.scriptFolder = scriptFolder;
}
- /**
- * Returns the folder which contains all Data SQL scripts. As a default value {@link #getScriptFolder()} will be
- * returned if not defined by a non-<code>null</code> value in {@link #setFolderOfDataScripts(String)}.
- *
- * @return <code>null</code> when {@link #getScriptFolder()} returns <code>null</code>.
- */
- public final String getFolderOfDataScripts()
- {
- return folderOfDataScripts == null ? getScriptFolder() : folderOfDataScripts;
- }
-
- /**
- * Sets the folder which contains all Data SQL scripts.
- *
- * @param folderOfDataScripts New value. Can be <code>null</code>.
- */
- public final void setFolderOfDataScripts(String folderOfDataScripts)
- {
- this.folderOfDataScripts = folderOfDataScripts;
- }
-
- /**
- * Returns the folder which contains the files for mass upload to the database. As a default value
- * {@link #getFolderOfDataScripts()} will be returned if not definied by a non-<code>null</code> value in
- * {@link #setFolderOfMassUploadFiles(String)}.
- */
- public String getFolderOfMassUploadFiles()
- {
- return folderOfMassUploadFiles == null ? getFolderOfDataScripts() : folderOfMassUploadFiles;
- }
-
- /**
- * Sets the folder which contains the files for mass uploads.
- *
- * @param folderOfMassUploadFiles New value. Can be <code>null</code>. An empty value will be interpreted as
- * <code>null</code>.
- */
- public void setFolderOfMassUploadFiles(String folderOfMassUploadFiles)
- {
- if ("".equals(folderOfMassUploadFiles))
- {
- this.folderOfMassUploadFiles = null;
- } else
- {
- this.folderOfMassUploadFiles = folderOfMassUploadFiles;
- }
- }
-
/** Closes opened database connections. */
public final void closeConnections()
{
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IDatabaseAdminDAO.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IDatabaseAdminDAO.java
index adf80d5f5ef..43acb9efa69 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IDatabaseAdminDAO.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IDatabaseAdminDAO.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.dbmigration;
+import java.io.File;
+
/**
* Interface for administration of a database.
*
@@ -35,14 +37,14 @@ public interface IDatabaseAdminDAO
public void createOwner();
/**
- * Creates the database.
+ * Creates the database and the 'database_version_logs' table.
*/
public void createDatabase();
/**
* Restores the database from previously created dump.
*/
- public void restoreDatabaseFromDump(ISqlScriptProvider scriptProvider, String version);
+ public void restoreDatabaseFromDump(File dumpFolder, String version);
/**
* Drops the database.
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/ISqlScriptProvider.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/ISqlScriptProvider.java
index 4de7c033ec8..3278f43d004 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/ISqlScriptProvider.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/ISqlScriptProvider.java
@@ -33,6 +33,11 @@ public interface ISqlScriptProvider
*/
public boolean isDumpRestore(String version);
+ /**
+ * Returns the folder where all dump files for <var>version</var> reside.
+ */
+ public File getDumpFolder(String version);
+
/**
* Returns the script to create database schemas.
*
@@ -57,14 +62,6 @@ public interface ISqlScriptProvider
*/
public Script tryGetDataScript(String version);
- /**
- * Returns the script to be executed to finish up database creation.
- *
- * @param version Version of the database.
- * @return <code>null</code> if there isn't such a script.
- */
- public Script tryGetFinishScript(String version);
-
/**
* Returns the migration script for migrating a database.
*
@@ -74,18 +71,4 @@ public interface ISqlScriptProvider
*/
public Script tryGetMigrationScript(String fromVersion, String toVersion);
- /**
- * Returns the script for creating the database log versions table.
- *
- * @return <code>null</code> if there isn't such a script.
- */
- public Script tryGetLogCreationScript();
-
- /**
- * Returns the files containing data for mass upload.
- *
- * @param version Version of the database.
- * @return The files to mass upload, or an empty array, if there are no such files.
- */
- public File[] getMassUploadFiles(String version);
}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/SqlScriptProvider.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/SqlScriptProvider.java
index 829211402b4..edbf56c57c1 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/SqlScriptProvider.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/SqlScriptProvider.java
@@ -17,9 +17,6 @@
package ch.systemsx.cisd.dbmigration;
import java.io.File;
-import java.io.FilenameFilter;
-import java.util.Arrays;
-
import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.Script;
@@ -36,69 +33,56 @@ import ch.systemsx.cisd.common.utilities.FileUtilities;
*/
public class SqlScriptProvider implements ISqlScriptProvider
{
- private static final String SQL_FILE_TYPE = ".sql";
+ private static final String DUMP_FILENAME = ".DUMP";
- //@Private
- static final String CREATE_LOG_SQL = "createLog.sql";
+ private static final String SQL_FILE_TYPE = ".sql";
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, SqlScriptProvider.class);
- private final String schemaScriptFolder;
-
- private final String dataScriptFolder;
-
- private final String massUploadDataFolder;
-
- private final String internalScriptFolder;
-
- /**
- * Creates an instance for the specified folders and database type. The database type specifies the resource folder
- * relative to the package of this class where the scripts with method {@link #tryGetLogCreationScript()} are loaded.
- */
- public static ISqlScriptProvider create(String schemaScriptFolder, String dataScriptFolder,
- String massUploadDataFolder, String databaseEngineCode)
- {
- final String internalFolder =
- SqlScriptProvider.class.getPackage().getName().replace('.', '/') + "/" + databaseEngineCode;
- return new SqlScriptProvider(schemaScriptFolder, dataScriptFolder, massUploadDataFolder, internalFolder);
- }
-
+ private final String genericScriptFolder;
+
+ private final String specificScriptFolder;
+
/**
* Creates an instance for the specified script folders. They are either resource folders or folders relative to the
* working directory.
*
- * @param schemaScriptFolder Folder of schema and migration scripts.
- * @param dataScriptFolder Folder of data scripts.
- * @param massUploadDataFolder Folder of CSV files for uploading.
- * @param internalScriptFolder Folder for internal scripts.
+ * @param schemaScriptRootFolder Root folder of schema, migration and data scripts.
+ * @param databaseEngineCode The code of the database engine. Used to find the db engine specific schema script
+ * folder.
*/
- SqlScriptProvider(String schemaScriptFolder, String dataScriptFolder, String massUploadDataFolder,
- String internalScriptFolder)
+ public SqlScriptProvider(String schemaScriptRootFolder, String databaseEngineCode)
{
- this.schemaScriptFolder = schemaScriptFolder;
- this.dataScriptFolder = dataScriptFolder;
- this.massUploadDataFolder = massUploadDataFolder;
- this.internalScriptFolder = internalScriptFolder;
+ this.genericScriptFolder = schemaScriptRootFolder + "/generic";
+ this.specificScriptFolder = schemaScriptRootFolder + "/" + databaseEngineCode;
}
-
+
/**
* Returns <code>true</code> if a <finish script> is found and <code>false</code> otherwise.
*/
public boolean isDumpRestore(String version)
{
- return tryGetFinishScript(version) != null;
+ return new File(getDumpFolder(version), DUMP_FILENAME).exists();
}
/**
- * Returns the data script for the specified version. The name of the script is expected to be
+ * Returns the folder where all dump files for <var>version</var> reside.
+ */
+ public File getDumpFolder(String version)
+ {
+ return new File(specificScriptFolder, version);
+ }
+
+ /**
+ * Returns the schema script for the specified version. The name of the script is expected to be
*
* <pre>
- * <data script folder>/<version>/finish-<version>.sql
+ * <schema script folder>/<version>/schema-<version>.sql
* </pre>
*/
- public Script tryGetFinishScript(String version)
+ public Script tryGetSchemaScript(String version)
{
- return tryLoadScript(schemaScriptFolder + "/" + version, "finish-" + version + SQL_FILE_TYPE, version);
+ return tryLoadScript("schema-" + version + SQL_FILE_TYPE, version);
}
/**
@@ -110,7 +94,7 @@ public class SqlScriptProvider implements ISqlScriptProvider
*/
public Script tryGetFunctionScript(String version)
{
- return tryLoadScript(schemaScriptFolder + "/" + version, "function-" + version + SQL_FILE_TYPE, version);
+ return tryLoadScript("function-" + version + SQL_FILE_TYPE, version);
}
/**
@@ -122,7 +106,7 @@ public class SqlScriptProvider implements ISqlScriptProvider
*/
public Script tryGetDataScript(String version)
{
- return tryLoadScript(dataScriptFolder + "/" + version, "data-" + version + SQL_FILE_TYPE, version);
+ return tryLoadScript("data-" + version + SQL_FILE_TYPE, version);
}
/**
@@ -134,38 +118,33 @@ public class SqlScriptProvider implements ISqlScriptProvider
*/
public Script tryGetMigrationScript(String fromVersion, String toVersion)
{
- String scriptName = "migration-" + fromVersion + "-" + toVersion + SQL_FILE_TYPE;
- return tryLoadScript(schemaScriptFolder + "/migration", scriptName, toVersion);
+ final String scriptName = "migration-" + fromVersion + "-" + toVersion + SQL_FILE_TYPE;
+ return tryLoadScript(scriptName, toVersion, "migration");
}
- /**
- * Returns the schema script for the specified version. The name of the script is expected to be
- *
- * <pre>
- * <schema script folder>/<version>/schema-<version>.sql
- * </pre>
- */
- public Script tryGetSchemaScript(String version)
+ private Script tryLoadScript(String scriptName, String scriptVersion)
{
- return tryLoadScript(schemaScriptFolder + "/" + version, "schema-" + version + SQL_FILE_TYPE, version);
+ return tryLoadScript(scriptName, scriptVersion, scriptVersion);
}
-
- /**
- * Returns the specified script relative to the internal script folder.
- */
- public Script tryGetLogCreationScript()
+
+ private Script tryLoadScript(String scriptName, String scriptVersion, String prefix)
{
- return tryLoadScript(internalScriptFolder, CREATE_LOG_SQL, "-");
+ Script script = tryPrimLoadScript(specificScriptFolder + "/" + prefix, scriptName, scriptVersion);
+ if (script == null)
+ {
+ script = tryPrimLoadScript(genericScriptFolder + "/" + prefix, scriptName, scriptVersion);
+ }
+ return script;
}
- private Script tryLoadScript(String folder, String scriptName, String scriptVersion)
+ private Script tryPrimLoadScript(String scriptFolder, String scriptName, String scriptVersion)
{
- String fullScriptName = folder + "/" + scriptName;
- String resource = "/" + fullScriptName;
+ final String scriptPath = scriptFolder + "/" + scriptName;
+ final String resource = "/" + scriptPath;
String script = FileUtilities.loadToString(getClass(), resource);
if (script == null)
{
- File file = new File(folder, scriptName);
+ File file = new File(scriptFolder, scriptName);
if (operationLog.isDebugEnabled())
{
operationLog.debug("Resource '" + resource + "' could not be found. Trying '" + file.getPath() + "'.");
@@ -180,42 +159,7 @@ public class SqlScriptProvider implements ISqlScriptProvider
}
script = FileUtilities.loadToString(file);
}
- return new Script(fullScriptName, script, scriptVersion);
- }
-
- /**
- * Returns the files determined for mass uploading.
- */
- public File[] getMassUploadFiles(String version)
- {
- final File dataFolder = new File(massUploadDataFolder + "/" + version);
- if (operationLog.isDebugEnabled())
- {
- operationLog.debug("Searching for mass upload files in directory '" + dataFolder.getAbsolutePath() + "'.");
- }
- String[] csvFiles = dataFolder.list(new FilenameFilter()
- {
- public boolean accept(File dir, String name)
- {
- return MassUploadFileType.CSV.isOfType(name) || MassUploadFileType.TSV.isOfType(name);
- }
- });
- if (csvFiles == null)
- {
- operationLog.warn("Path '" + dataFolder.getAbsolutePath() + "' is not a directory.");
- return new File[0];
- }
- Arrays.sort(csvFiles);
- if (operationLog.isInfoEnabled())
- {
- operationLog.info("Found " + csvFiles.length + " files for mass uploading.");
- }
- final File[] csvPaths = new File[csvFiles.length];
- for (int i = 0; i < csvFiles.length; ++i)
- {
- csvPaths[i] = new File(dataFolder, csvFiles[i]);
- }
- return csvPaths;
+ return new Script(scriptPath, script, scriptVersion);
}
}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/postgresql/PostgreSQLAdminDAO.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/postgresql/PostgreSQLAdminDAO.java
index 03fb740da52..eb03058bc6a 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/postgresql/PostgreSQLAdminDAO.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/postgresql/PostgreSQLAdminDAO.java
@@ -17,12 +17,13 @@
package ch.systemsx.cisd.dbmigration.postgresql;
import java.io.File;
+import java.io.FilenameFilter;
+import java.util.Arrays;
import javax.sql.DataSource;
import org.apache.log4j.Logger;
import org.springframework.dao.DataAccessException;
-import org.springframework.jdbc.BadSqlGrammarException;
import org.springframework.jdbc.core.simple.SimpleJdbcDaoSupport;
import ch.systemsx.cisd.common.Script;
@@ -30,10 +31,11 @@ import ch.systemsx.cisd.common.db.ISqlScriptExecutor;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.utilities.FileUtilities;
import ch.systemsx.cisd.dbmigration.DBUtilities;
import ch.systemsx.cisd.dbmigration.IDatabaseAdminDAO;
import ch.systemsx.cisd.dbmigration.IMassUploader;
-import ch.systemsx.cisd.dbmigration.ISqlScriptProvider;
+import ch.systemsx.cisd.dbmigration.MassUploadFileType;
/**
* Implementation of {@link IDatabaseAdminDAO} for PostgreSQL.
@@ -42,6 +44,17 @@ import ch.systemsx.cisd.dbmigration.ISqlScriptProvider;
*/
public class PostgreSQLAdminDAO extends SimpleJdbcDaoSupport implements IDatabaseAdminDAO
{
+ private static final String SQL_FILE_TYPE = ".sql";
+
+ private static final String CREATE_DATABASE_SQL_TEMPLATE =
+ "create database %1$s with owner = %2$s encoding = 'utf8' tablespace = pg_default; "
+ + "alter database %1$s set default_with_oids = off;";
+
+ private static final String CREATE_TABLE_DATABASE_VERSION_LOGS_SQL =
+ "create table database_version_logs (db_version varchar(4) not null, "
+ + "module_name varchar(250), run_status varchar(10), run_status_timestamp timestamp, "
+ + "module_code bytea, run_exception bytea);";
+
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, PostgreSQLAdminDAO.class);
private final ISqlScriptExecutor scriptExecutor;
@@ -57,7 +70,7 @@ public class PostgreSQLAdminDAO extends SimpleJdbcDaoSupport implements IDatabas
*
* @param dataSource Data source able to create/drop the specified database.
* @param scriptExecutor An executor for SQL scripts.
- * @param massUploader A class that can perform mass (batch) uploads into database tables.
+ * @param massUploader A class that can perform mass (batch) uploads into database tables.
* @param owner Owner to be created if it doesn't exist.
* @param database Name of the database.
*/
@@ -102,21 +115,40 @@ public class PostgreSQLAdminDAO extends SimpleJdbcDaoSupport implements IDatabas
}
public void createDatabase()
+ {
+ createEmptyDatabase();
+ createDatabaseVersionLogsTable();
+ }
+
+ private void createDatabaseVersionLogsTable()
+ {
+ try
+ {
+ scriptExecutor.execute(new Script("create database_version_logs table",
+ CREATE_TABLE_DATABASE_VERSION_LOGS_SQL), true, null);
+ } catch (RuntimeException ex)
+ {
+ operationLog.error("Failed to create database_version_logs table.", ex);
+ throw ex;
+ }
+ }
+
+ private void createEmptyDatabase()
{
operationLog.info("Try to create empty database '" + database + "' with owner '" + owner + "'.");
try
{
- getJdbcTemplate().execute(
- "create database " + database + " with owner = " + owner
- + " encoding = 'utf8' tablespace = pg_default;" + "alter database " + database
- + " set default_with_oids = off;");
- } catch (BadSqlGrammarException ex)
+ getJdbcTemplate().execute(String.format(CREATE_DATABASE_SQL_TEMPLATE, database, owner));
+ } catch (RuntimeException ex)
{
- if (DBUtilities.isDuplicateDatabaseException(ex) == false)
+ if (ex instanceof DataAccessException && DBUtilities.isDuplicateDatabaseException((DataAccessException) ex))
{
+ operationLog.warn("Cannot create database '" + database + "' since it already exists.");
+ } else
+ {
+ operationLog.error("Failed to create database '" + database + "'.", ex);
throw ex;
}
- operationLog.warn("Cannot create database '" + database + "' since it already exists.");
}
}
@@ -134,27 +166,64 @@ public class PostgreSQLAdminDAO extends SimpleJdbcDaoSupport implements IDatabas
}
}
- public void restoreDatabaseFromDump(ISqlScriptProvider scriptProvider, String version)
+ public void restoreDatabaseFromDump(File dumpFolder, String version)
{
- createDatabase();
- final Script schemaScript = scriptProvider.tryGetSchemaScript(version);
- if (schemaScript == null)
- {
- final String message = "No schema script found for version " + version;
- operationLog.error(message);
- throw new ConfigurationFailureException(message);
- }
+ createEmptyDatabase();
+
+ final Script schemaScript = tryLoadScript(dumpFolder, "schema", version);
scriptExecutor.execute(schemaScript, false, null);
- final File[] massUploadFiles = scriptProvider.getMassUploadFiles(version);
+ final File[] massUploadFiles = getMassUploadFiles(dumpFolder);
massUploader.performMassUpload(massUploadFiles);
- final Script finishScript = scriptProvider.tryGetFinishScript(version);
- if (schemaScript == null)
+ final Script finishScript = tryLoadScript(dumpFolder, "finish", version);
+ scriptExecutor.execute(finishScript, false, null);
+ }
+
+ private Script tryLoadScript(final File dumpFolder, String prefix, String version)
+ throws ConfigurationFailureException
+ {
+ final File scriptFile = new File(dumpFolder, prefix + "-" + version + SQL_FILE_TYPE);
+ final Script script = new Script(scriptFile.getPath(), FileUtilities.loadToString(scriptFile), version);
+ if (script == null)
{
- final String message = "No finish script found for version " + version;
+ final String message = "No " + prefix + " script found for version " + version;
operationLog.error(message);
throw new ConfigurationFailureException(message);
}
- scriptExecutor.execute(finishScript, false, null);
+ return script;
+ }
+
+ /**
+ * Returns the files determined for mass uploading.
+ */
+ private File[] getMassUploadFiles(File dumpFolder)
+ {
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug("Searching for mass upload files in directory '" + dumpFolder.getAbsolutePath() + "'.");
+ }
+ String[] csvFiles = dumpFolder.list(new FilenameFilter()
+ {
+ public boolean accept(File dir, String name)
+ {
+ return MassUploadFileType.CSV.isOfType(name) || MassUploadFileType.TSV.isOfType(name);
+ }
+ });
+ if (csvFiles == null)
+ {
+ operationLog.warn("Path '" + dumpFolder.getAbsolutePath() + "' is not a directory.");
+ return new File[0];
+ }
+ Arrays.sort(csvFiles);
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Found " + csvFiles.length + " files for mass uploading.");
+ }
+ final File[] csvPaths = new File[csvFiles.length];
+ for (int i = 0; i < csvFiles.length; ++i)
+ {
+ csvPaths[i] = new File(dumpFolder, csvFiles[i]);
+ }
+ return csvPaths;
}
}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/postgresql/createLog.sql b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/postgresql/createLog.sql
deleted file mode 100644
index ec427f34e13..00000000000
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/postgresql/createLog.sql
+++ /dev/null
@@ -1,8 +0,0 @@
-create table database_version_logs
- (db_version varchar(4) not null
- ,module_name varchar(250)
- ,run_status varchar(10)
- ,run_status_timestamp timestamp
- ,module_code bytea
- ,run_exception bytea
- );
\ No newline at end of file
diff --git a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/DBMigrationEngineTest.java b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/DBMigrationEngineTest.java
index eb31240540c..6619390b854 100644
--- a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/DBMigrationEngineTest.java
+++ b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/DBMigrationEngineTest.java
@@ -20,8 +20,7 @@ import static org.testng.AssertJUnit.assertEquals;
import static org.testng.AssertJUnit.assertSame;
import static org.testng.AssertJUnit.fail;
-import java.io.PrintWriter;
-import java.io.StringWriter;
+import java.io.File;
import java.sql.SQLException;
import java.util.Date;
@@ -35,6 +34,7 @@ import org.testng.annotations.Test;
import ch.systemsx.cisd.common.Script;
import ch.systemsx.cisd.common.db.ISqlScriptExecutor;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.logging.BufferedAppender;
import ch.systemsx.cisd.common.utilities.OSUtilities;
@@ -53,14 +53,6 @@ public class DBMigrationEngineTest
will(returnValue(script));
one(scriptExecutor).execute(script, honorSingleStepMode, logDAO);
}
-
- protected void expectCreateLogDAOTable()
- {
- one(scriptProvider).tryGetLogCreationScript();
- Script script = new Script(SqlScriptProvider.CREATE_LOG_SQL, "create log");
- will(returnValue(script));
- one(logDAO).createTable(with(same(script)));
- }
}
private Mockery context;
@@ -131,7 +123,6 @@ public class DBMigrationEngineTest
will(returnValue(false));
one(adminDAO).createDatabase();
- expectCreateLogDAOTable();
one(scriptProvider).tryGetSchemaScript(version);
expectSuccessfulExecution(new Script("schema", "schema code", version), true);
one(scriptProvider).tryGetFunctionScript(version);
@@ -175,7 +166,10 @@ public class DBMigrationEngineTest
one(scriptProvider).isDumpRestore(version);
will(returnValue(true));
- one(adminDAO).restoreDatabaseFromDump(scriptProvider, version);
+ one(scriptProvider).getDumpFolder(version);
+ final File dumpFolder = new File("The Dump Folder");
+ will(returnValue(dumpFolder));
+ one(adminDAO).restoreDatabaseFromDump(dumpFolder, version);
one(adminDAO).getDatabaseName();
will(returnValue("my 2. database"));
}
@@ -251,7 +245,6 @@ public class DBMigrationEngineTest
one(scriptProvider).isDumpRestore(version);
will(returnValue(false));
one(adminDAO).createDatabase();
- expectCreateLogDAOTable();
one(scriptProvider).tryGetSchemaScript(version);
expectSuccessfulExecution(new Script("schema", "schema code"), true);
one(scriptProvider).tryGetFunctionScript(version);
@@ -295,7 +288,6 @@ public class DBMigrationEngineTest
will(returnValue(false));
one(adminDAO).createDatabase();
- expectCreateLogDAOTable();
one(scriptProvider).tryGetSchemaScript(version);
expectSuccessfulExecution(new Script("schema", "schema code", version), true);
one(scriptProvider).tryGetFunctionScript(version);
@@ -338,7 +330,6 @@ public class DBMigrationEngineTest
one(scriptProvider).isDumpRestore(version);
will(returnValue(false));
one(adminDAO).createDatabase();
- expectCreateLogDAOTable();
one(scriptProvider).tryGetSchemaScript(version);
will(returnValue(null));
}
@@ -349,50 +340,7 @@ public class DBMigrationEngineTest
{
migrationEngine.migrateTo(version);
fail("EnvironmentFailureException expected because of missing schema script");
- } catch (EnvironmentFailureException e)
- {
- assertEquals(message, e.getMessage());
- }
- assertEquals("INFO OPERATION.ch.systemsx.cisd.dbmigration.DBMigrationEngine - "
- + "Database 'my 1. database' does not exist." + OSUtilities.LINE_SEPARATOR
- + "ERROR OPERATION.ch.systemsx.cisd.dbmigration.DBMigrationEngine - " + message, logRecorder
- .getLogContent());
-
- context.assertIsSatisfied();
- }
-
- @Test
- public void testCreateFromScratchWithMissingCreateLogScript()
- {
- final String version = "042";
- context.checking(new MyExpectations()
- {
- {
- one(daoFactory).getDatabaseDAO();
- will(returnValue(adminDAO));
- one(daoFactory).getDatabaseVersionLogDAO();
- will(returnValue(logDAO));
- one(daoFactory).getSqlScriptExecutor();
- will(returnValue(scriptExecutor));
-
- one(logDAO).canConnectToDatabase();
- will(returnValue(false));
- one(adminDAO).getDatabaseName();
- will(returnValue("my 1. database"));
- one(adminDAO).createOwner();
- one(scriptProvider).isDumpRestore(version);
- will(returnValue(false));
- one(adminDAO).createDatabase();
- one(scriptProvider).tryGetLogCreationScript();
- }
- });
- DBMigrationEngine migrationEngine = new DBMigrationEngine(daoFactory, scriptProvider, false);
- String message = "Missing log creation script";
- try
- {
- migrationEngine.migrateTo(version);
- fail("EnvironmentFailureException expected because of missing log creation script");
- } catch (EnvironmentFailureException e)
+ } catch (ConfigurationFailureException e)
{
assertEquals(message, e.getMessage());
}
@@ -404,51 +352,6 @@ public class DBMigrationEngineTest
context.assertIsSatisfied();
}
- @Test
- public void testCreateFromScratchWithCreateLogScriptWhichFails()
- {
- final String version = "042";
- final RuntimeException exception = new RuntimeException();
- context.checking(new MyExpectations()
- {
- {
- one(daoFactory).getDatabaseDAO();
- will(returnValue(adminDAO));
- one(daoFactory).getDatabaseVersionLogDAO();
- will(returnValue(logDAO));
- one(daoFactory).getSqlScriptExecutor();
- will(returnValue(scriptExecutor));
-
- one(logDAO).canConnectToDatabase();
- will(returnValue(false));
- one(scriptProvider).isDumpRestore(version);
- will(returnValue(false));
- one(adminDAO).getDatabaseName();
- will(returnValue("my database"));
- one(adminDAO).createOwner();
- one(adminDAO).createDatabase();
- expectCreateLogDAOTable();
- will(throwException(exception));
- }
- });
- DBMigrationEngine migrationEngine = new DBMigrationEngine(daoFactory, scriptProvider, false);
- try
- {
- migrationEngine.migrateTo(version);
- fail("RuntimeException expected because of failing log creation script");
- } catch (RuntimeException e)
- {
- assertSame(exception, e);
- }
- assertEquals("INFO OPERATION.ch.systemsx.cisd.dbmigration.DBMigrationEngine - "
- + "Database 'my database' does not exist." + OSUtilities.LINE_SEPARATOR
- + "ERROR OPERATION.ch.systemsx.cisd.dbmigration.DBMigrationEngine - Script 'createLog.sql' failed:"
- + OSUtilities.LINE_SEPARATOR + "create log" + OSUtilities.LINE_SEPARATOR + getStackTrace(exception),
- logRecorder.getLogContent());
-
- context.assertIsSatisfied();
- }
-
@Test
public void testCaseNoMigrationNeeded()
{
@@ -795,11 +698,4 @@ public class DBMigrationEngineTest
context.assertIsSatisfied();
}
- private String getStackTrace(final Throwable throwable)
- {
- StringWriter stringWriter = new StringWriter();
- throwable.printStackTrace(new PrintWriter(stringWriter));
- return stringWriter.toString().trim();
- }
-
}
diff --git a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/SqlScriptProviderTest.java b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/SqlScriptProviderTest.java
index 0f54d5498b0..06cae24c25d 100644
--- a/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/SqlScriptProviderTest.java
+++ b/dbmigration/sourceTest/java/ch/systemsx/cisd/dbmigration/SqlScriptProviderTest.java
@@ -17,19 +17,21 @@
package ch.systemsx.cisd.dbmigration;
import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertFalse;
import static org.testng.AssertJUnit.assertNotNull;
import static org.testng.AssertJUnit.assertTrue;
-
import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.PrintWriter;
+import org.apache.commons.io.IOUtils;
import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.Test;
import ch.systemsx.cisd.common.Script;
+import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
/**
* Tests for {@link SqlScriptExecutor}.
@@ -41,28 +43,17 @@ public class SqlScriptProviderTest
private static final String BASE_FOLDER =
"targets" + File.separator + "unit-test-wd" + File.separator + "SqlScriptProviderTest";
- private static final String TEMPORARY_DATA_SCRIPT_FOLDER_NAME =
- BASE_FOLDER + File.separator + "temporaryDataScriptFolder";
-
private static final String TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME =
BASE_FOLDER + File.separator + "temporarySchemaScriptFolder";
- private static final String TEMPORARY_MASS_DATA_UPLOAD_FOLDER_NAME =
- BASE_FOLDER + File.separator + "temporaryMassDataUploadFolder";
-
- private static final String TEMPORARY_INTERNAL_SCRIPT_FOLDER_NAME =
- BASE_FOLDER + File.separator + "temporaryInternalScriptFolder";
-
- private static final File TEMP_SCHEMA_SCRIPT_FOLDER = new File(TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME);
-
- private static final File TEMP_DATA_SCRIPT_FOLDER = new File(TEMPORARY_DATA_SCRIPT_FOLDER_NAME);
+ private static final String DB_ENGINE_CODE = "dbengine";
- private static final File TEMP_MASS_DATA_UPLOAD_FOLDER = new File(TEMPORARY_MASS_DATA_UPLOAD_FOLDER_NAME);
+ private static final File TEMP_SCHEMA_SCRIPT_ROOT_FOLDER = new File(TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME);
- private static final File TEMP_INTERNAL_SCRIPT_FOLDER = new File(TEMPORARY_INTERNAL_SCRIPT_FOLDER_NAME);
+ private static final File TEMP_SCHEMA_GENERIC_SCRIPT_FOLDER = new File(TEMP_SCHEMA_SCRIPT_ROOT_FOLDER, "generic");
- private static final File CREATE_LOG_SQL_FILE =
- new File(TEMP_INTERNAL_SCRIPT_FOLDER, SqlScriptProvider.CREATE_LOG_SQL);
+ private static final File TEMP_SCHEMA_SPECIFIC_SCRIPT_FOLDER =
+ new File(TEMP_SCHEMA_SCRIPT_ROOT_FOLDER, DB_ENGINE_CODE);
private static final String MIGRATION = "migration";
@@ -72,52 +63,60 @@ public class SqlScriptProviderTest
private SqlScriptProvider sqlScriptProvider;
+ private File dumpFile;
+
@BeforeClass
- public void setUpTestFiles() throws IOException
- {
- File schemaVersionFolder = new File(TEMP_SCHEMA_SCRIPT_FOLDER, VERSION);
- schemaVersionFolder.mkdirs();
- write(new File(schemaVersionFolder, "schema-" + VERSION + ".sql"), "code: schema");
- write(new File(schemaVersionFolder, "function-" + VERSION + ".sql"), "code: function");
- write(new File(schemaVersionFolder, "finish-" + VERSION + ".sql"), "code: finish");
- File migrationFolder = new File(TEMP_SCHEMA_SCRIPT_FOLDER, MIGRATION);
+ public void setUpTestFiles()
+ {
+ final File genericSchemaVersionFolder = createGenericSchemaFolder();
+ final File specificSchemaVersionFolder = createSpecificSchemaFolder();
+ genericSchemaVersionFolder.mkdirs();
+ specificSchemaVersionFolder.mkdirs();
+ write(new File(specificSchemaVersionFolder, "schema-" + VERSION + ".sql"), "code: schema");
+ write(new File(specificSchemaVersionFolder, "function-" + VERSION + ".sql"), "code: function");
+ final File migrationFolder = new File(TEMP_SCHEMA_SPECIFIC_SCRIPT_FOLDER, MIGRATION);
migrationFolder.mkdir();
write(new File(migrationFolder, "migration-" + VERSION + "-" + VERSION2 + ".sql"), "code: migration");
- File dataVersionFolder = new File(TEMP_DATA_SCRIPT_FOLDER, VERSION);
- dataVersionFolder.mkdirs();
- write(new File(dataVersionFolder, "data-" + VERSION + ".sql"), "code: data");
- TEMP_INTERNAL_SCRIPT_FOLDER.mkdir();
- File massUploaadVersionFolder = new File(TEMP_MASS_DATA_UPLOAD_FOLDER, VERSION);
- massUploaadVersionFolder.mkdirs();
- write(new File(massUploaadVersionFolder, "1=test.tsv"), "1\tbla");
- sqlScriptProvider =
- new SqlScriptProvider(TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME, TEMPORARY_DATA_SCRIPT_FOLDER_NAME,
- TEMPORARY_MASS_DATA_UPLOAD_FOLDER_NAME, TEMPORARY_INTERNAL_SCRIPT_FOLDER_NAME);
+ write(new File(specificSchemaVersionFolder, "data-" + VERSION + ".sql"), "code: data");
+ sqlScriptProvider = new SqlScriptProvider(TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME, DB_ENGINE_CODE);
+ dumpFile = new File(sqlScriptProvider.getDumpFolder(VERSION), ".DUMP");
+ }
+
+ private File createSpecificSchemaFolder()
+ {
+ final File specificSchemaVersionFolder = new File(TEMP_SCHEMA_SPECIFIC_SCRIPT_FOLDER, VERSION);
+ return specificSchemaVersionFolder;
}
- private void write(File file, String content) throws IOException
+ private File createGenericSchemaFolder()
+ {
+ final File specificSchemaVersionFolder = new File(TEMP_SCHEMA_GENERIC_SCRIPT_FOLDER, VERSION);
+ return specificSchemaVersionFolder;
+ }
+
+ private void write(File file, String content)
{
- FileWriter fileWriter = null;
try
{
- fileWriter = new FileWriter(file);
- new PrintWriter(fileWriter).print(content);
- } finally
- {
- if (fileWriter != null)
+ FileWriter fileWriter = null;
+ try
{
- fileWriter.close();
+ fileWriter = new FileWriter(file);
+ new PrintWriter(fileWriter).print(content);
+ } finally
+ {
+ IOUtils.closeQuietly(fileWriter);
}
+ } catch (IOException ex)
+ {
+ throw new CheckedExceptionTunnel(ex);
}
}
@AfterClass
public void deleteTestFiles()
{
- delete(TEMP_SCHEMA_SCRIPT_FOLDER);
- delete(TEMP_DATA_SCRIPT_FOLDER);
- delete(TEMP_INTERNAL_SCRIPT_FOLDER);
- delete(TEMP_MASS_DATA_UPLOAD_FOLDER);
+ delete(TEMP_SCHEMA_SCRIPT_ROOT_FOLDER);
}
private void delete(File file)
@@ -138,11 +137,28 @@ public class SqlScriptProviderTest
{
final Script script = sqlScriptProvider.tryGetSchemaScript(VERSION);
assertNotNull(script);
- assertEquals(TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME + "/" + VERSION + "/schema-" + VERSION + ".sql", script
- .getName());
+ assertEquals(TEMP_SCHEMA_SPECIFIC_SCRIPT_FOLDER.getPath() + "/" + VERSION + "/schema-" + VERSION + ".sql",
+ script.getName());
assertEquals("code: schema", script.getCode().trim());
}
+ // Note: we make it dependent on testGetSchemaScript(), because we delete the specific schema script
+ // in this test case and thus testGetSchemaScript() would fail if run after this test case.
+ @Test(dependsOnMethods="testGetSchemaScript")
+ public void testGetGenericSchemaScript()
+ {
+ final File specificSchemaScript = new File(createSpecificSchemaFolder(), "schema-" + VERSION + ".sql");
+ specificSchemaScript.delete();
+ final File genericSchemaScript = new File(createGenericSchemaFolder(), "schema-" + VERSION + ".sql");
+ final String genericSchemaScriptContent = "code: generic schema";
+ write(genericSchemaScript, genericSchemaScriptContent);
+ final Script script = sqlScriptProvider.tryGetSchemaScript(VERSION);
+ assertNotNull(script);
+ assertEquals(TEMP_SCHEMA_GENERIC_SCRIPT_FOLDER.getPath() + "/" + VERSION + "/schema-" + VERSION + ".sql",
+ script.getName());
+ assertEquals(genericSchemaScriptContent, script.getCode().trim());
+ }
+
@Test
public void testGetNonExistingSchemaScript()
{
@@ -154,8 +170,8 @@ public class SqlScriptProviderTest
{
final Script script = sqlScriptProvider.tryGetFunctionScript(VERSION);
assertNotNull(script);
- assertEquals(TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME + "/" + VERSION + "/function-" + VERSION + ".sql", script
- .getName());
+ assertEquals(TEMP_SCHEMA_SPECIFIC_SCRIPT_FOLDER.getPath() + "/" + VERSION + "/function-" + VERSION + ".sql",
+ script.getName());
assertEquals("code: function", script.getCode().trim());
}
@@ -165,39 +181,14 @@ public class SqlScriptProviderTest
assertEquals(null, sqlScriptProvider.tryGetFunctionScript("000"));
}
- @Test
- public void testGetFinishScript()
- {
- final Script script = sqlScriptProvider.tryGetFinishScript(VERSION);
- assertNotNull(script);
- assertEquals(TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME + "/" + VERSION + "/finish-" + VERSION + ".sql", script
- .getName());
- assertEquals("code: finish", script.getCode().trim());
- }
-
- @Test
- public void testGetNonExistingFinishScript()
- {
- assertEquals(null, sqlScriptProvider.tryGetFinishScript("000"));
- }
-
@Test
public void testGetDataScript()
{
Script script = sqlScriptProvider.tryGetDataScript(VERSION);
- assertEquals(TEMPORARY_DATA_SCRIPT_FOLDER_NAME + "/" + VERSION + "/data-" + VERSION + ".sql", script.getName());
+ assertEquals(TEMP_SCHEMA_SPECIFIC_SCRIPT_FOLDER + "/" + VERSION + "/data-" + VERSION + ".sql", script.getName());
assertEquals("code: data", script.getCode().trim());
}
- @Test
- void testGetMassUploadFiles()
- {
- final File[] massUploadFiles = sqlScriptProvider.getMassUploadFiles(VERSION);
- assertEquals(1, massUploadFiles.length);
- assertEquals("1=test.tsv", massUploadFiles[0].getName());
- assertTrue(massUploadFiles[0].exists());
- }
-
@Test
public void testGetNonExistingDataScript()
{
@@ -208,8 +199,8 @@ public class SqlScriptProviderTest
public void testGetMigrationScript()
{
Script script = sqlScriptProvider.tryGetMigrationScript(VERSION, VERSION2);
- assertEquals(TEMPORARY_SCHEMA_SCRIPT_FOLDER_NAME + "/" + MIGRATION + "/migration-" + VERSION + "-" + VERSION2
- + ".sql", script.getName());
+ assertEquals(TEMP_SCHEMA_SPECIFIC_SCRIPT_FOLDER.getPath() + "/" + MIGRATION + "/migration-" + VERSION + "-"
+ + VERSION2 + ".sql", script.getName());
assertEquals("code: migration", script.getCode().trim());
}
@@ -220,20 +211,17 @@ public class SqlScriptProviderTest
}
@Test
- public void testGetCreateLogScript() throws IOException
+ public void testIsDumpSucceeds()
{
- write(CREATE_LOG_SQL_FILE, "hello world!");
- Script script = sqlScriptProvider.tryGetLogCreationScript();
- assertNotNull(script);
- assertEquals(TEMPORARY_INTERNAL_SCRIPT_FOLDER_NAME + "/" + SqlScriptProvider.CREATE_LOG_SQL, script.getName());
- assertEquals("hello world!", script.getCode().trim());
+ write(dumpFile, "");
+ assertTrue(sqlScriptProvider.isDumpRestore(VERSION));
}
@Test
- public void testGetNonExistingCreateLogScript()
+ public void testIsDumpFails()
{
- CREATE_LOG_SQL_FILE.delete();
- assertEquals(null, sqlScriptProvider.tryGetLogCreationScript());
+ dumpFile.delete();
+ assertFalse(sqlScriptProvider.isDumpRestore(VERSION));
}
}
--
GitLab