LMS-2655 - Create a unit test to check imaging db consistency

SVN: 23647

dbmigration/.classpath

@@ -22,5 +22,6 @@
 	<classpathentry kind="lib" path="/libraries/jmock/objenesis/objenesis-1.0.jar"/>
 	<classpathentry kind="lib" path="/libraries/cisd-base/cisd-base-test.jar" sourcepath="/libraries/cisd-base/cisd-base-src.zip"/>
 	<classpathentry kind="lib" path="/libraries/cisd-base/cisd-base.jar" sourcepath="/libraries/cisd-base/cisd-base-src.zip"/>
+	<classpathentry kind="lib" path="/libraries/apgdiff/apgdiff.jar" sourcepath="/libraries/apgdiff/src.zip"/>
 	<classpathentry kind="output" path="targets/classes"/>
 </classpath>

dbmigration/source/java/ch/systemsx/cisd/dbmigration/migration/SqlMigrationDataSource.java

+package ch.systemsx.cisd.dbmigration.migration;
+
+import java.io.PrintWriter;
+import java.sql.Connection;
+import java.sql.DriverManager;
+import java.sql.SQLException;
+
+import javax.sql.DataSource;
+
+import org.springframework.beans.factory.DisposableBean;
+
+public class SqlMigrationDataSource implements DataSource, DisposableBean
+{
+    private final String driver;
+
+    private final String url;
+
+    private final String owner;
+
+    private final String password;
+
+    private Connection connection;
+
+    SqlMigrationDataSource(final String driver, final String url, final String owner,
+            final String password)
+    {
+        this.driver = driver;
+        this.url = url;
+        this.owner = owner;
+        this.password = password;
+    }
+
+    public Connection getConnection() throws SQLException
+    {
+        if (connection != null && connection.isClosed() || connection == null)
+        {
+            try
+            {
+                Class.forName(driver);
+            } catch (final ClassNotFoundException ex)
+            {
+                throw new SQLException("Couldn't load driver " + driver);
+            }
+            final Connection c = DriverManager.getConnection(url, owner, password);
+            connection = c;
+        }
+        return connection;
+    }
+
+    public Connection getConnection(final String username, final String pw) throws SQLException
+    {
+        if (owner.equals(username) && password.equals(pw))
+        {
+            return getConnection();
+        }
+        throw new SQLException("Forbidden");
+    }
+
+    public int getLoginTimeout() throws SQLException
+    {
+        return 0;
+    }
+
+    public void setLoginTimeout(final int timeout) throws SQLException
+    {
+        throw new UnsupportedOperationException("setLoginTimeout");
+    }
+
+    public PrintWriter getLogWriter()
+    {
+        throw new UnsupportedOperationException("getLogWriter");
+    }
+
+    public void setLogWriter(final PrintWriter pw) throws SQLException
+    {
+        throw new UnsupportedOperationException("setLogWriter");
+    }
+
+    public void destroy() throws SQLException
+    {
+        if (connection != null)
+        {
+            connection.close();
+            connection = null;
+        }
+    }
+
+    @Override
+    public String toString()
+    {
+        return "MyDataSource[" + driver + ", " + url + ", " + owner + "]";
+    }
+
+    // NOTE: the following methods are needed because we compile with JDK 6 on Hudson
+    public boolean isWrapperFor(Class<?> arg0) throws SQLException
+    {
+        return false;
+    }
+
+    public <T> T unwrap(Class<T> arg0) throws SQLException
+    {
+        return null;
+    }
+    
+}
\ No newline at end of file

dbmigration/source/java/ch/systemsx/cisd/dbmigration/migration/SqlMigrationDataSourceFactory.java

+package ch.systemsx.cisd.dbmigration.migration;
+
+import javax.sql.DataSource;
+
+import ch.systemsx.cisd.dbmigration.IDataSourceFactory;
+
+public class SqlMigrationDataSourceFactory implements IDataSourceFactory
+{
+
+    public DataSource createDataSource(String driver, String url, String owner, String password)
+    {
+        return new SqlMigrationDataSource(driver, url, owner, password);
+    }
+
+    public void setMaxActive(int maxActive)
+    {
+    }
+
+    public void setMaxIdle(int maxIdle)
+    {
+    }
+
+    public void setMaxWait(long maxWait)
+    {
+    }
+
+}
\ No newline at end of file

dbmigration/source/java/ch/systemsx/cisd/dbmigration/migration/SqlMigrationTestAbstract.java

+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.dbmigration.migration;
+
+import java.io.File;
+import java.io.PrintWriter;
+import java.io.StringWriter;
+
+import org.testng.AssertJUnit;
+import org.testng.annotations.AfterTest;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeTest;
+
+import cz.startnet.utils.pgdiff.PgDiff;
+import cz.startnet.utils.pgdiff.PgDiffArguments;
+
+import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.dbmigration.DBMigrationEngine;
+import ch.systemsx.cisd.dbmigration.DatabaseConfigurationContext;
+import ch.systemsx.cisd.dbmigration.postgresql.DumpPreparator;
+
+/**
+ * Test cases for database migration.
+ * 
+ * @author Piotr Kupczyk
+ */
+public abstract class SqlMigrationTestAbstract
+{
+
+    private File sqlScriptOutputDirectory;
+
+    protected abstract String getSqlScriptInputDirectory();
+
+    protected abstract String getSqlScriptOutputDirectory();
+
+    @BeforeClass(alwaysRun = true)
+    public void beforeClass() throws Exception
+    {
+        LogInitializer.init();
+    }
+
+    @BeforeTest(alwaysRun = true)
+    public void beforeTest() throws Exception
+    {
+        File dir = new File(getSqlScriptOutputDirectory());
+        if (!dir.exists())
+        {
+            dir.mkdir();
+        }
+    }
+
+    @AfterTest(alwaysRun = true)
+    public void afterTest() throws Exception
+    {
+        File dir = new File(getSqlScriptOutputDirectory());
+        if (dir.exists())
+        {
+            dir.delete();
+        }
+    }
+
+    public void test_migration(String firstVersion, String newestVersion) throws Exception
+    {
+        int firstVersionInt = Integer.valueOf(firstVersion);
+        int newestVersionInt = Integer.valueOf(newestVersion);
+
+        DatabaseConfigurationContext migrationContext = null;
+        DatabaseConfigurationContext scratchContext = null;
+
+        try
+        {
+
+            migrationContext = createMigrationDatabaseContext(true);
+            scratchContext = createScratchDatabaseContext();
+
+            // create first version of migration database
+            DBMigrationEngine.createOrMigrateDatabaseAndGetScriptProvider(migrationContext,
+                    firstVersion);
+            
+            migrationContext.setCreateFromScratch(false);
+
+            for (int version = firstVersionInt + 1; version <= newestVersionInt; version++)
+            {
+                String versionStr = String.format("%03d", version);
+
+                // migrate to the next version
+                DBMigrationEngine.createOrMigrateDatabaseAndGetScriptProvider(migrationContext,
+                        versionStr);
+                dumpDatabaseSchema(migrationContext, getMigratedDatabaseSchemaFile());
+
+                // create next version from scratch
+                DBMigrationEngine.createOrMigrateDatabaseAndGetScriptProvider(scratchContext,
+                        versionStr);
+                dumpDatabaseSchema(scratchContext, getScratchDatabaseSchemaFile());
+
+                // check whether migrated and scratch version are equal
+                assertDatabaseSchemasEqual(getMigratedDatabaseSchemaFile(),
+                        getScratchDatabaseSchemaFile());
+            }
+
+        } finally
+        {
+            if (migrationContext != null)
+            {
+                migrationContext.closeConnections();
+            }
+            if (scratchContext != null)
+            {
+                scratchContext.closeConnections();
+            }
+        }
+    }
+
+    private DatabaseConfigurationContext createDatabaseContext(String dbKind,
+            boolean createFromScratch)
+    {
+        DatabaseConfigurationContext context = new DatabaseConfigurationContext();
+        context.setDatabaseEngineCode("postgresql");
+        context.setBasicDatabaseName("openbis");
+        context.setDatabaseKind(dbKind);
+        context.setScriptFolder(getSqlScriptInputDirectory());
+        context.initDataSourceFactory(new SqlMigrationDataSourceFactory());
+        context.setCreateFromScratch(createFromScratch);
+        return context;
+    }
+
+    private DatabaseConfigurationContext createMigrationDatabaseContext(boolean createFromScratch)
+    {
+        return createDatabaseContext("test_migration_migrated", createFromScratch);
+    }
+
+    private DatabaseConfigurationContext createScratchDatabaseContext()
+    {
+        return createDatabaseContext("test_migration_scratch", true);
+    }
+
+    private File getMigratedDatabaseSchemaFile()
+    {
+        return new File(sqlScriptOutputDirectory, "migratedDatabaseSchema.sql");
+    }
+
+    private File getScratchDatabaseSchemaFile()
+    {
+        return new File(sqlScriptOutputDirectory, "scratchDatabaseSchema.sql");
+    }
+
+    private void dumpDatabaseSchema(final DatabaseConfigurationContext configurationContext,
+            final File migratedSchemaFile)
+    {
+        final boolean dumpSuccessful =
+                DumpPreparator.createDatabaseSchemaDump(configurationContext.getDatabaseName(),
+                        migratedSchemaFile);
+        AssertJUnit.assertTrue("dump of db failed: " + configurationContext.getDatabaseName(),
+                dumpSuccessful);
+    }
+
+    private void assertDatabaseSchemasEqual(final File currentSchemaFile,
+            final File expectedSchemaFile)
+    {
+        final StringWriter writer = new StringWriter();
+        final PgDiffArguments arguments = new PgDiffArguments();
+        arguments.setOldDumpFile(currentSchemaFile.getAbsolutePath());
+        arguments.setNewDumpFile(expectedSchemaFile.getAbsolutePath());
+        arguments.setIgnoreFunctionWhitespace(true);
+        arguments.setIgnoreStartWith(true);
+        PgDiff.createDiff(new PrintWriter(writer), arguments);
+
+        String delta = writer.toString();
+        delta = delta == null ? "" : delta;
+
+        AssertJUnit.assertEquals("The migrated schema is not identical to the scratch one. "
+                + "Consider attaching following script to the migration file.", "", delta);
+    }
+
+}

screening/.classpath

@@ -51,5 +51,6 @@
 	<classpathentry kind="lib" path="/libraries/gxt2.2.5/gxt.jar"/>
 	<classpathentry kind="lib" path="/libraries/gwt2.4/validation-api-1.0.0.GA-sources.jar"/>
 	<classpathentry kind="lib" path="/libraries/gwt2.4/validation-api-1.0.0.GA.jar"/>
+	<classpathentry kind="lib" path="/libraries/apgdiff/apgdiff.jar" sourcepath="/libraries/apgdiff/src.zip"/>
 	<classpathentry kind="output" path="targets/www/WEB-INF/classes"/>
 </classpath>

screening/source/java/ch/systemsx/cisd/openbis/dss/generic/server/dataaccess/db/ScreeningSqlMigrationTest.java

+/*
+ * Copyright 2007 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.server.dataaccess.db;
+
+import java.io.File;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.dbmigration.migration.SqlMigrationTestAbstract;
+import ch.systemsx.cisd.openbis.dss.etl.ImagingDatabaseVersionHolder;
+
+/**
+ * Test cases for database migration.
+ * 
+ * @author Piotr Kupczyk
+ */
+public class ScreeningSqlMigrationTest extends SqlMigrationTestAbstract
+{
+
+    @Test(groups =
+        { "slow" })
+    public void test_migration()
+            throws Exception
+    {
+        test_migration("001",
+                new ImagingDatabaseVersionHolder().getDatabaseVersion());
+    }
+
+    @Override
+    protected String getSqlScriptInputDirectory()
+    {
+        return "source" + File.separator + "sql" + File.separator + "imaging";
+    }
+
+    @Override
+    protected String getSqlScriptOutputDirectory()
+    {
+        return "targets" + File.separator + "unit-test-wd";
+    }
+
+}

screening/source/sql/imaging/postgresql/002/schema-002.sql

@@ -182,15 +182,15 @@ CREATE TABLE FEATURE_VALUES (
 		VALUES BYTEA NOT NULL,
 		
 		FD_ID  TECH_ID NOT NULL,
-		DS_ID  TECH_ID NOT NULL,
+		--DS_ID  TECH_ID NOT NULL,
 		
 		PRIMARY KEY (ID),
-		CONSTRAINT FK_FEATURE_VALUES_1 FOREIGN KEY (FD_ID) REFERENCES FEATURE_DEFS (ID) ON DELETE CASCADE ON UPDATE CASCADE,
-    CONSTRAINT FK_FEATURE_VALUES_2 FOREIGN KEY (DS_ID) REFERENCES DATA_SETS (ID) ON DELETE CASCADE ON UPDATE CASCADE
+		CONSTRAINT FK_FEATURE_VALUES_1 FOREIGN KEY (FD_ID) REFERENCES FEATURE_DEFS (ID) ON DELETE CASCADE ON UPDATE CASCADE
+    --CONSTRAINT FK_FEATURE_VALUES_2 FOREIGN KEY (DS_ID) REFERENCES DATA_SETS (ID) ON DELETE CASCADE ON UPDATE CASCADE
     -- This constaint does not make any sense. Leave it out for now.
     -- CONSTRAINT FEATURE_VALUES_UK_1 UNIQUE(Z_in_M, T_in_SEC)
 );
 
 CREATE INDEX FEATURE_VALUES_FD_IDX ON FEATURE_VALUES(FD_ID);
-CREATE INDEX FEATURE_VALUES_DS_IDX ON FEATURE_VALUES(DS_ID);
+--CREATE INDEX FEATURE_VALUES_DS_IDX ON FEATURE_VALUES(DS_ID);
 CREATE INDEX FEATURE_VALUES_Z_AND_T_IDX ON FEATURE_VALUES(Z_in_M, T_in_SEC);

screening/source/sql/imaging/postgresql/migration/migration-016-017.sql

@@ -25,6 +25,7 @@ ALTER TABLE ACQUIRED_IMAGES DROP CONSTRAINT FK_IMAGES_3;
 ALTER TABLE ACQUIRED_IMAGES ADD CONSTRAINT FK_IMAGES_3 FOREIGN KEY (IMG_ID) REFERENCES IMAGES (ID) ON DELETE SET NULL ON UPDATE CASCADE;
 ALTER TABLE ACQUIRED_IMAGES DROP CONSTRAINT FK_IMAGES_4;
 ALTER TABLE ACQUIRED_IMAGES ADD CONSTRAINT FK_IMAGES_4 FOREIGN KEY (THUMBNAIL_ID) REFERENCES IMAGES (ID) ON DELETE SET NULL ON UPDATE CASCADE;
+ALTER TABLE acquired_images	ADD CONSTRAINT fk_images_img_or_thumb_arc_ck CHECK (IMG_ID IS NOT NULL OR THUMBNAIL_ID IS NOT NULL);
 
 CREATE OR REPLACE FUNCTION DELETE_UNUSED_NULLED_IMAGES() RETURNS trigger AS $$
 BEGIN