diff --git a/dbmigration/.classpath b/dbmigration/.classpath
index 5812838ec1b968657952372964240683cfb618b6..1e9f32ebbdda0aa5c5f5cbd861820c52f86f765d 100644
--- a/dbmigration/.classpath
+++ b/dbmigration/.classpath
@@ -1,19 +1,21 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<classpath>
- <classpathentry kind="src" path="source/java"/>
- <classpathentry kind="src" path="sourceTest/java"/>
- <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
- <classpathentry kind="lib" path="/libraries/spring/spring.jar" sourcepath="/libraries/spring/src.zip"/>
- <classpathentry combineaccessrules="false" kind="src" path="/common"/>
- <classpathentry kind="lib" path="/libraries/log4j/log4j.jar" sourcepath="/libraries/log4j/src.zip"/>
- <classpathentry kind="lib" path="/libraries/commons-lang/commons-lang.jar" sourcepath="/libraries/commons-lang/src.zip"/>
- <classpathentry kind="lib" path="/libraries/commons-dbcp/commons-dbcp.jar" sourcepath="/libraries/commons-dbcp/src.zip"/>
- <classpathentry kind="lib" path="/libraries/testng/testng-jdk15.jar" sourcepath="/libraries/testng/src.zip"/>
- <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/cglib-2.1_3-src.jar"/>
- <classpathentry kind="lib" path="/libraries/jmock/jmock.jar"/>
- <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/cglib-nodep-2.1_3.jar"/>
- <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/hamcrest-api-1.0.jar"/>
- <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/hamcrest-library-1.0.jar"/>
- <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/objenesis-1.0.jar"/>
- <classpathentry kind="output" path="targets/classes"/>
-</classpath>
+<?xml version="1.0" encoding="UTF-8"?>
+<classpath>
+ <classpathentry kind="src" path="source/java"/>
+ <classpathentry kind="src" path="sourceTest/java"/>
+ <classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
+ <classpathentry kind="lib" path="/libraries/spring/spring.jar" sourcepath="/libraries/spring/src.zip"/>
+ <classpathentry combineaccessrules="false" kind="src" path="/common"/>
+ <classpathentry kind="lib" path="/libraries/log4j/log4j.jar" sourcepath="/libraries/log4j/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/commons-lang/commons-lang.jar" sourcepath="/libraries/commons-lang/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/commons-dbcp/commons-dbcp.jar" sourcepath="/libraries/commons-dbcp/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/testng/testng-jdk15.jar" sourcepath="/libraries/testng/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/cglib-2.1_3-src.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/jmock.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/cglib-nodep-2.1_3.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/hamcrest-api-1.0.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/hamcrest-library-1.0.jar"/>
+ <classpathentry kind="lib" path="/libraries/jmock/third-party-libs/objenesis-1.0.jar"/>
+ <classpathentry kind="lib" path="/libraries/postgresql/postgresql.jar" sourcepath="/libraries/postgresql/src.zip"/>
+ <classpathentry kind="lib" path="/libraries/commons-io/commons-io.jar" sourcepath="/libraries/commons-io/src.zip"/>
+ <classpathentry kind="output" path="targets/classes"/>
+</classpath>
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DBMigrationEngine.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DBMigrationEngine.java
index f38c2cca4d9f345aa664d4c98d83ae2b0beebba4..8aa7e638df883dd49107fedf0f27ba4081d9a5cd 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DBMigrationEngine.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/DBMigrationEngine.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.dbmigration;
+import java.io.File;
+
import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
@@ -43,6 +45,8 @@ public class DBMigrationEngine
private final IDatabaseAdminDAO adminDAO;
private final IDatabaseVersionLogDAO logDAO;
+
+ private final IMassUploader massUploader;
private final ISqlScriptExecutor scriptExecutor;
@@ -55,6 +59,7 @@ public class DBMigrationEngine
{
adminDAO = daoFactory.getDatabaseDAO();
logDAO = daoFactory.getDatabaseVersionLogDAO();
+ massUploader = daoFactory.getMassUploader();
scriptExecutor = daoFactory.getSqlScriptExecutor();
this.scriptProvider = scriptProvider;
this.shouldCreateFromScratch = shouldCreateFromScratch;
@@ -179,6 +184,11 @@ public class DBMigrationEngine
{
executeScript(initialDataScript, version);
}
+ File[] massUploadFiles = scriptProvider.getMassUploadFiles(version);
+ for (File f : massUploadFiles)
+ {
+ massUploader.performMassUpload(f);
+ }
}
private void migrate(String fromVersion, String toVersion)
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IDAOFactory.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IDAOFactory.java
index cbaf2a4fd496e2637891905051851a6af91e2632..9603a058ac2629de0b8ba47e886ed91f43dd8c48 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IDAOFactory.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IDAOFactory.java
@@ -37,4 +37,6 @@ public interface IDAOFactory
* Returns the DAO for the database version log.
*/
public IDatabaseVersionLogDAO getDatabaseVersionLogDAO();
+
+ public IMassUploader getMassUploader();
}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IMassUploader.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IMassUploader.java
new file mode 100644
index 0000000000000000000000000000000000000000..243474ebcfb1fc67ca7f4283f6e989f212655b74
--- /dev/null
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/IMassUploader.java
@@ -0,0 +1,41 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.dbmigration;
+
+import java.io.File;
+
+/**
+ * Classes that implement this interface encapsulate mass uploading of tabular files to the database. The files to be
+ * processed by implementations of this class are supposed to be <code>csv</code> files that can be uploaded to
+ * exactly one table. They need to follow the naming convention <code>orderSpecifier::tableName.csv</code>. Here the
+ * order specifier ensures that the dependency is right while the table name is the name of the table to upload the data
+ * to.
+ *
+ * @author Bernd Rinn
+ */
+public interface IMassUploader
+{
+
+ /**
+ * Upload the data from file <var>massUploadFile</var> to the database.
+ *
+ * @param massUploadFile File to upload to the database, following the naming convention
+ * <code>orderSpecifier::tableName.csv</code>.
+ */
+ public void performMassUpload(File massUploadFile);
+
+}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/ISqlScriptProvider.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/ISqlScriptProvider.java
index 1442bef42c3e40c8bc0c50361201c292b140e425..cab0513cc81766a07223eee28ce7357372b37a12 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/ISqlScriptProvider.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/ISqlScriptProvider.java
@@ -16,6 +16,8 @@
package ch.systemsx.cisd.dbmigration;
+import java.io.File;
+
/**
* Provider of SQL scripts for creation and migration of database.
*
@@ -55,4 +57,11 @@ public interface ISqlScriptProvider
* @return <code>null</code> if there isn't such a script.
*/
public Script getScript(String scriptName);
+
+ /**
+ * Returns the files containing data for mass upload.
+ * @param version Version of the database.
+ * @return The files to mass upload, or an empty array, if there are no such files.
+ */
+ public File[] getMassUploadFiles(String version);
}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/PostgreSQLDAOFactory.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/PostgreSQLDAOFactory.java
index a3c7f976c35eb3f4c8b65cadf7777f66a9b577b4..03ea5eb6770b28be2a706d95d8bbd1f335aeda0f 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/PostgreSQLDAOFactory.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/PostgreSQLDAOFactory.java
@@ -16,8 +16,12 @@
package ch.systemsx.cisd.dbmigration;
+import java.sql.SQLException;
+
import javax.sql.DataSource;
+import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
+
/**
* Implementation of {@link IDAOFactory} for PostgreSQL.
*
@@ -28,6 +32,7 @@ public class PostgreSQLDAOFactory implements IDAOFactory
private final IDatabaseAdminDAO databaseDAO;
private final ISqlScriptExecutor sqlScriptExecutor;
private final IDatabaseVersionLogDAO databaseVersionLogDAO;
+ private final IMassUploader massUploader;
/**
* Creates an instance based on the specified configuration context.
@@ -38,6 +43,13 @@ public class PostgreSQLDAOFactory implements IDAOFactory
final DataSource dataSource = context.getDataSource();
sqlScriptExecutor = new SqlScriptExecutor(dataSource);
databaseVersionLogDAO = new DatabaseVersionLogDAO(dataSource, context.getLobHandler());
+ try
+ {
+ massUploader = new PostgreSQLMassUploader(dataSource);
+ } catch (SQLException ex)
+ {
+ throw new CheckedExceptionTunnel(ex);
+ }
}
public IDatabaseAdminDAO getDatabaseDAO()
@@ -55,4 +67,9 @@ public class PostgreSQLDAOFactory implements IDAOFactory
return databaseVersionLogDAO;
}
+ public IMassUploader getMassUploader()
+ {
+ return massUploader;
+ }
+
}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/PostgreSQLMassUploader.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/PostgreSQLMassUploader.java
new file mode 100644
index 0000000000000000000000000000000000000000..d5002edd0e0fb080fe0cb30b462aea12f5551653
--- /dev/null
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/PostgreSQLMassUploader.java
@@ -0,0 +1,127 @@
+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.dbmigration;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.lang.reflect.Field;
+import java.sql.Connection;
+import java.sql.SQLException;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
+import org.apache.log4j.Logger;
+import org.postgresql.PGConnection;
+import org.postgresql.copy.CopyManager;
+
+import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+
+/**
+ * A {@link IMassUploader} for the PostgreSQL database.
+ *
+ * @author Bernd Rinn
+ */
+public class PostgreSQLMassUploader implements IMassUploader
+{
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, PostgreSQLMassUploader.class);
+
+ private final DataSource dataSource;
+
+ public PostgreSQLMassUploader(DataSource dataSource) throws SQLException
+ {
+ this.dataSource = dataSource;
+ }
+
+ public void performMassUpload(File massUploadFile)
+ {
+ try
+ {
+ final CopyManager copyManager = getPGConnection().getCopyAPI();
+ final String[] splitName = StringUtils.split(massUploadFile.getName(), "::");
+ assert splitName.length == 2;
+ final String tableNameWithExtension = splitName[1];
+ assert tableNameWithExtension.endsWith(".csv");
+ final String tableName =
+ tableNameWithExtension.substring(0, tableNameWithExtension.length() - ".csv".length());
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info("Perform mass upload of file '" + massUploadFile + "' to table '" + tableName + "'.");
+ }
+ final InputStream is = new FileInputStream(massUploadFile);
+ try
+ {
+ copyManager.copyInQuery("COPY " + tableName + " FROM STDIN WITH CSV HEADER", is);
+ } finally
+ {
+ IOUtils.closeQuietly(is);
+ }
+ } catch (Exception ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ private PGConnection getPGConnection() throws SQLException, NoSuchFieldException, IllegalAccessException
+ {
+ return getPGConnection(dataSource.getConnection());
+ }
+
+ private PGConnection getPGConnection(Connection conn) throws SQLException, NoSuchFieldException,
+ IllegalAccessException
+ {
+ if (conn instanceof PGConnection)
+ {
+ return (PGConnection) conn;
+ }
+ if (operationLog.isDebugEnabled())
+ {
+ operationLog.debug("Found connection of type '" + conn.getClass().getCanonicalName() + "'.");
+ }
+ Field delegateField = getField(conn.getClass(), "_conn");
+ if (delegateField == null)
+ {
+ throw new RuntimeException("No PostgreSQL driver found - cannot perform mass upload.");
+ }
+ delegateField.setAccessible(true);
+ return getPGConnection((Connection) delegateField.get(conn));
+ }
+
+ private static Field getField(Class<?> clazz, String fieldName)
+ {
+ assert fieldName != null;
+ if (clazz == null)
+ {
+ return null;
+ }
+
+ for (Field field : clazz.getDeclaredFields())
+ {
+ if (fieldName.equals(field.getName()))
+ {
+ return field;
+ }
+ }
+ return getField(clazz.getSuperclass(), fieldName);
+ }
+
+}
diff --git a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/SqlScriptProvider.java b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/SqlScriptProvider.java
index 94234e426802dd761dcb8230dc922d33c1d9e5df..10615a1bd59a49cd07162638a675c827c30de9e3 100644
--- a/dbmigration/source/java/ch/systemsx/cisd/dbmigration/SqlScriptProvider.java
+++ b/dbmigration/source/java/ch/systemsx/cisd/dbmigration/SqlScriptProvider.java
@@ -17,6 +17,8 @@
package ch.systemsx.cisd.dbmigration;
import java.io.File;
+import java.io.FilenameFilter;
+import java.util.Arrays;
import org.apache.log4j.Logger;
@@ -131,5 +133,31 @@ public class SqlScriptProvider implements ISqlScriptProvider
return new Script(fullScriptName, script);
}
+ /**
+ * Returns the files determined for mass uploading.
+ */
+ public File[] getMassUploadFiles(String version)
+ {
+ final File dataFolder = new File(dataScriptFolder + "/" + version);
+ String[] csvFiles = dataFolder.list(new FilenameFilter()
+ {
+ public boolean accept(File dir, String name)
+ {
+ return name.endsWith(".csv");
+ }
+ });
+ Arrays.sort(csvFiles);
+ if (csvFiles == null)
+ {
+ return new File[0];
+ }
+ final File[] csvPaths = new File[csvFiles.length];
+ for (int i = 0; i < csvFiles.length; ++i)
+ {
+ csvPaths[i] = new File(dataFolder, csvFiles[i]);
+ }
+ return csvPaths;
+ }
+
}