From c3410fe5dcc5efc9562b46d15b0efe6b2825a3d5 Mon Sep 17 00:00:00 2001 From: felmer <felmer> Date: Tue, 15 May 2012 08:45:02 +0000 Subject: [PATCH] SP-39 reading and checking report database meta data. Tests added. SVN: 25256 --- .../server/task/MaterialReportingTask.java | 165 +++++++++++++++--- .../generic/shared/util/DataTypeUtils.java | 33 ++++ .../cisd/openbis/plugin/query/server/DAO.java | 35 +--- .../task/MaterialReportingTaskTest.java | 82 +++++++-- 4 files changed, 253 insertions(+), 62 deletions(-) diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTask.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTask.java index 335b625110d..a39774bd1a8 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTask.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTask.java @@ -17,23 +17,33 @@ package ch.systemsx.cisd.openbis.generic.server.task; import java.io.File; +import java.sql.DatabaseMetaData; import java.sql.PreparedStatement; +import java.sql.ResultSet; import java.sql.SQLException; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Map.Entry; import java.util.Properties; +import java.util.Set; import java.util.TreeMap; import org.apache.log4j.Logger; import org.springframework.jdbc.core.BatchPreparedStatementSetter; +import org.springframework.jdbc.core.ColumnMapRowMapper; import org.springframework.jdbc.core.JdbcTemplate; +import org.springframework.jdbc.support.DatabaseMetaDataCallback; +import org.springframework.jdbc.support.JdbcUtils; +import org.springframework.jdbc.support.MetaDataAccessException; import ch.rinn.restrictions.Private; import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; +import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; import ch.systemsx.cisd.common.filesystem.FileUtilities; import ch.systemsx.cisd.common.logging.LogCategory; import ch.systemsx.cisd.common.logging.LogFactory; @@ -43,6 +53,7 @@ import ch.systemsx.cisd.dbmigration.SimpleDatabaseConfigurationContext; import ch.systemsx.cisd.openbis.generic.server.CommonServiceProvider; import ch.systemsx.cisd.openbis.generic.server.ICommonServerForInternalUse; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.CompareType; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataTypeCode; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchCriteria; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchCriterion; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchField; @@ -51,6 +62,7 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Material; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialAttributeSearchFieldKind; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SearchCriteriaConnection; import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO; +import ch.systemsx.cisd.openbis.generic.shared.util.DataTypeUtils; /** * Task which feeds a reporting database with recently added/changed Materials. @@ -65,13 +77,15 @@ public class MaterialReportingTask implements IMaintenanceTask @Private static final class MappingInfo { - private static final class NameAndIndex + private static final class Column { private final String name; private final int index; - NameAndIndex(String name, int index) + private DataTypeCode dataTypeCode; + + Column(String name, int index) { this.name = name; this.index = index; @@ -84,8 +98,7 @@ public class MaterialReportingTask implements IMaintenanceTask private final String codeColumnName; - private final Map<String, NameAndIndex> propertyMapping = - new TreeMap<String, NameAndIndex>(); + private final Map<String, Column> propertyMapping = new TreeMap<String, Column>(); MappingInfo(String materialTypeCode, String tableName, String codeColumnName) { @@ -94,17 +107,54 @@ public class MaterialReportingTask implements IMaintenanceTask this.codeColumnName = codeColumnName; } + String getTableName() + { + return tableName; + } + + String getCodeColumnName() + { + return codeColumnName; + } + void addPropertyMapping(String propertyTypeCode, String propertyColumnName) { - propertyMapping.put(propertyTypeCode, new NameAndIndex(propertyColumnName, - propertyMapping.size())); + Column column = new Column(propertyColumnName, propertyMapping.size()); + propertyMapping.put(propertyTypeCode, column); + } + + void injectDataTypeCodes(Map<String, DataTypeCode> columns) + { + DataTypeCode codeColumnType = columns.remove(codeColumnName); + if (codeColumnType == null) + { + throw new EnvironmentFailureException("Missing column '" + codeColumnName + + "' in table '" + tableName + "' of report database."); + } + if (codeColumnType.equals(DataTypeCode.VARCHAR) == false) + { + throw new EnvironmentFailureException("Column '" + codeColumnName + "' of table '" + + tableName + "' is not of type VARCHAR."); + } + Collection<Column> values = propertyMapping.values(); + for (Column column : values) + { + DataTypeCode dataTypeCode = columns.get(column.name); + if (dataTypeCode == null) + { + throw new EnvironmentFailureException("Missing column '" + column.name + + "' in table '" + tableName + "' of report database."); + } + column.dataTypeCode = dataTypeCode; + } + } String createInsertStatement() { StringBuilder builder = new StringBuilder("insert into ").append(tableName); builder.append(" (").append(codeColumnName); - for (NameAndIndex nameAndIndex : propertyMapping.values()) + for (Column nameAndIndex : propertyMapping.values()) { builder.append(", ").append(nameAndIndex.name); } @@ -133,7 +183,7 @@ public class MaterialReportingTask implements IMaintenanceTask for (IEntityProperty property : properties) { String code = property.getPropertyType().getCode(); - NameAndIndex nameAndIndex = propertyMapping.get(code); + Column nameAndIndex = propertyMapping.get(code); if (nameAndIndex != null) { ps.setObject(nameAndIndex.index + 2, property.tryGetAsString()); @@ -158,6 +208,8 @@ public class MaterialReportingTask implements IMaintenanceTask private SimpleDatabaseConfigurationContext dbConfigurationContext; + private JdbcTemplate jdbcTemplate; + public MaterialReportingTask() { this(CommonServiceProvider.getCommonServer()); @@ -177,6 +229,68 @@ public class MaterialReportingTask implements IMaintenanceTask // "write-timestamp-sql"); String mappingFileName = PropertyUtils.getMandatoryProperty(properties, MAPPING_FILE_KEY); mapping = readMappingFile(mappingFileName); + Map<String, Map<String, DataTypeCode>> metaData = retrieveDatabaseMetaData(); + for (MappingInfo mappingInfo : mapping.values()) + { + String tableName = mappingInfo.getTableName(); + Map<String, DataTypeCode> columns = metaData.get(tableName); + if (columns == null) + { + throw new EnvironmentFailureException("Missing table '" + tableName + + "' in report database."); + } + mappingInfo.injectDataTypeCodes(columns); + } + jdbcTemplate = new JdbcTemplate(dbConfigurationContext.getDataSource()); + } + + private Map<String, Map<String, DataTypeCode>> retrieveDatabaseMetaData() + { + Collection<MappingInfo> values = mapping.values(); + final Set<String> tableNames = new HashSet<String>(); + for (MappingInfo mappingInfo : values) + { + tableNames.add(mappingInfo.getTableName()); + } + try + { + final Map<String, Map<String, DataTypeCode>> map = + new HashMap<String, Map<String, DataTypeCode>>(); + JdbcUtils.extractDatabaseMetaData(dbConfigurationContext.getDataSource(), + new DatabaseMetaDataCallback() + { + + public Object processMetaData(DatabaseMetaData metaData) + throws SQLException, MetaDataAccessException + { + ResultSet rs = metaData.getColumns(null, null, null, null); + while (rs.next()) + { + String tableName = rs.getString("TABLE_NAME").toLowerCase(); + if (tableNames.contains(tableName)) + { + Map<String, DataTypeCode> columns = map.get(tableName); + if (columns == null) + { + columns = new TreeMap<String, DataTypeCode>(); + map.put(tableName, columns); + } + String columnName = + rs.getString("COLUMN_NAME").toLowerCase(); + DataTypeCode dataTypeCode = + DataTypeUtils.getDataTypeCode(rs + .getInt("DATA_TYPE")); + columns.put(columnName, dataTypeCode); + } + } + return null; + } + }); + return map; + } catch (MetaDataAccessException ex) + { + throw new ConfigurationFailureException("Couldn't retrieve meta data of database.", ex); + } } @Private @@ -197,21 +311,29 @@ public class MaterialReportingTask implements IMaintenanceTask } String sessionToken = contextOrNull.getSessionToken(); - JdbcTemplate jdbcTemplate = new JdbcTemplate(dbConfigurationContext.getDataSource()); Map<String, List<Material>> materialsByType = getRecentlyAddedOrChangedMaterials(sessionToken); for (Entry<String, List<Material>> entry : materialsByType.entrySet()) { String materialTypeCode = entry.getKey(); final List<Material> materials = entry.getValue(); - MappingInfo mappingInfo = mapping.get(materialTypeCode); - if (mappingInfo != null) - { - String insertStatement = mappingInfo.createInsertStatement(); - jdbcTemplate.batchUpdate(insertStatement, mappingInfo.createSetter(materials)); - } + addAndUpdate(materialTypeCode, materials); + } + } + + private void addAndUpdate(String materialTypeCode, final List<Material> materials) + { + MappingInfo mappingInfo = mapping.get(materialTypeCode); + if (mappingInfo != null) + { + List<?> rows = + jdbcTemplate.query("select * from " + mappingInfo.getTableName(), + new ColumnMapRowMapper()); + String insertStatement = mappingInfo.createInsertStatement(); + jdbcTemplate.batchUpdate(insertStatement, mappingInfo.createSetter(materials)); + operationLog.info(materials.size() + " materials of type " + materialTypeCode + + " reported."); } - operationLog.info(materialsByType.size() + " materials reported."); } private Map<String, List<Material>> getRecentlyAddedOrChangedMaterials(String sessionToken) @@ -220,7 +342,7 @@ public class MaterialReportingTask implements IMaintenanceTask DetailedSearchCriterion criterion = new DetailedSearchCriterion( DetailedSearchField - .createAttributeField(MaterialAttributeSearchFieldKind.MODIFICATION_DATE_UNTIL), + .createAttributeField(MaterialAttributeSearchFieldKind.MODIFICATION_DATE), CompareType.MORE_THAN_OR_EQUAL, readTimestamp(), "0"); criteria.setCriteria(Arrays.asList(criterion)); criteria.setConnection(SearchCriteriaConnection.MATCH_ALL); @@ -242,7 +364,7 @@ public class MaterialReportingTask implements IMaintenanceTask private String readTimestamp() { - return "2012-02-22"; + return "2012-02-22 10:33:44.6667"; } @Private @@ -274,14 +396,17 @@ public class MaterialReportingTask implements IMaintenanceTask splittedLine = splitAndCheck(splittedLine[1], ",", 2, factory); String tableName = trimeAndCheck(splittedLine[0], factory, "table name"); String codeColumnName = trimeAndCheck(splittedLine[1], factory, "code column name"); - currentMappingInfo = new MappingInfo(materialTypeCode, tableName, codeColumnName); + currentMappingInfo = + new MappingInfo(materialTypeCode, tableName.toLowerCase(), + codeColumnName.toLowerCase()); map.put(materialTypeCode, currentMappingInfo); } else if (currentMappingInfo != null) { String[] splittedLine = splitAndCheck(line, ":", 2, factory); String propertyTypeCode = trimeAndCheck(splittedLine[0], factory, "property type code"); - currentMappingInfo.addPropertyMapping(propertyTypeCode, splittedLine[1].trim()); + currentMappingInfo.addPropertyMapping(propertyTypeCode, splittedLine[1].trim() + .toLowerCase()); } else { throw factory.exception("Missing first material type table definition of form " diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/util/DataTypeUtils.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/util/DataTypeUtils.java index 1a7e977240d..fd88ede018c 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/util/DataTypeUtils.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/util/DataTypeUtils.java @@ -17,6 +17,7 @@ package ch.systemsx.cisd.openbis.generic.shared.util; import java.io.Serializable; +import java.sql.Types; import java.util.HashMap; import java.util.Map; @@ -193,6 +194,38 @@ public class DataTypeUtils return DataTypeCode.VARCHAR; } + /** + * Translates {@link java.sql.Types} codes into {@link DataTypeCode}. + */ + public static DataTypeCode getDataTypeCode(int sqlType) + { + if (isInteger(sqlType)) + { + return DataTypeCode.INTEGER; + } + if (isReal(sqlType)) + { + return DataTypeCode.REAL; + } + if (Types.DATE == sqlType || Types.TIMESTAMP == sqlType) + { + return DataTypeCode.TIMESTAMP; + } + return DataTypeCode.VARCHAR; + } + + private static boolean isInteger(int sqlType) + { + return Types.BIGINT == sqlType || Types.INTEGER == sqlType || Types.SMALLINT == sqlType + || Types.TINYINT == sqlType; + } + + private static boolean isReal(int sqlType) + { + return Types.DECIMAL == sqlType || Types.DOUBLE == sqlType || Types.FLOAT == sqlType + || Types.NUMERIC == sqlType || Types.REAL == sqlType; + } + private DataTypeUtils() { } diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/plugin/query/server/DAO.java b/openbis/source/java/ch/systemsx/cisd/openbis/plugin/query/server/DAO.java index e85007e1af8..a381470ab0a 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/plugin/query/server/DAO.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/plugin/query/server/DAO.java @@ -20,7 +20,6 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.sql.ResultSetMetaData; import java.sql.SQLException; -import java.sql.Types; import java.util.ArrayList; import java.util.Date; import java.util.HashMap; @@ -39,7 +38,6 @@ import org.springframework.jdbc.support.JdbcUtils; import ch.systemsx.cisd.common.exceptions.UserFailureException; import ch.systemsx.cisd.common.utilities.Counters; import ch.systemsx.cisd.common.utilities.Template; -import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataTypeCode; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DateTableCell; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DoubleTableCell; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityKind; @@ -49,6 +47,7 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.StringTableCell; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModel; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModelColumnHeader; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TableModelRow; +import ch.systemsx.cisd.openbis.generic.shared.util.DataTypeUtils; import ch.systemsx.cisd.openbis.plugin.query.shared.basic.dto.QueryParameterBindings; /** @@ -83,35 +82,6 @@ class DAO extends SimpleJdbcDaoSupport implements IDAO return entityKindByColumnName.get(columnName.toUpperCase()); } - private static DataTypeCode getDataTypeCode(int sqlType) - { - if (isInteger(sqlType)) - { - return DataTypeCode.INTEGER; - } - if (isReal(sqlType)) - { - return DataTypeCode.REAL; - } - if (Types.DATE == sqlType || Types.TIMESTAMP == sqlType) - { - return DataTypeCode.TIMESTAMP; - } - return DataTypeCode.VARCHAR; - } - - private static boolean isInteger(int sqlType) - { - return Types.BIGINT == sqlType || Types.INTEGER == sqlType || Types.SMALLINT == sqlType - || Types.TINYINT == sqlType; - } - - private static boolean isReal(int sqlType) - { - return Types.DECIMAL == sqlType || Types.DOUBLE == sqlType || Types.FLOAT == sqlType - || Types.NUMERIC == sqlType || Types.REAL == sqlType; - } - public DAO(DataSource dataSource) { setDataSource(dataSource); @@ -162,7 +132,8 @@ class DAO extends SimpleJdbcDaoSupport implements IDAO } TableModelColumnHeader header = new TableModelColumnHeader(columnName, id, i - 1); - header.setDataType(getDataTypeCode(metaData.getColumnType(i))); + header.setDataType(DataTypeUtils.getDataTypeCode(metaData + .getColumnType(i))); header.setEntityKind(entityKindOrNull); headers.add(header); } diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTaskTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTaskTest.java index 591e380dc59..7ad1e3ca7f8 100644 --- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTaskTest.java +++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTaskTest.java @@ -36,6 +36,7 @@ import org.testng.annotations.Test; import ch.rinn.restrictions.Friend; import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase; import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; +import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; import ch.systemsx.cisd.common.filesystem.FileUtilities; import ch.systemsx.cisd.common.test.RecordingMatcher; import ch.systemsx.cisd.dbmigration.DatabaseConfigurationContext; @@ -67,6 +68,8 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase private File mappingFile; + private Properties properties; + @BeforeMethod public void setUpMocks() throws Exception { @@ -82,10 +85,16 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase databaseName = dbConfigContext.getDatabaseName(); dropTestDatabase(); createTestDatabase(); - createTables("create table report1 (code varchar(20), description varchar(200))", + createTables( + "create table report1 (id bigint, code varchar(20), description varchar(200))", "create table report2 (code varchar(20), prop1 varchar(200), prop2 varchar(200))"); dbConfigContext.closeConnections(); mappingFile = new File(workingDirectory, "mapping-file.txt"); + properties = new Properties(); + properties.setProperty("database-driver", "org.postgresql.Driver"); + properties.setProperty("database-url", "jdbc:postgresql://localhost/" + databaseName); + properties.setProperty("database-username", "postgres"); + properties.setProperty(MaterialReportingTask.MAPPING_FILE_KEY, mappingFile.getPath()); } @AfterMethod @@ -100,15 +109,15 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase public void testReadValidMappingFile() throws SQLException { FileUtilities.writeToFile(mappingFile, "# my mapping\n[ T1 : TABLE1 , CODE ]\n\n" - + "[T2: TABLE2, code]\n P2 : prop2 \n P1 : prop1 "); + + "[T2: TABLE2, code]\n P2 : Prop2 \n P1 : PROP1 "); Map<String, MappingInfo> mapping = MaterialReportingTask.readMappingFile(mappingFile.getPath()); MappingInfo mappingInfo1 = mapping.get("T1"); - assertEquals("insert into TABLE1 (CODE) values(?)", mappingInfo1.createInsertStatement()); + assertEquals("insert into table1 (code) values(?)", mappingInfo1.createInsertStatement()); MappingInfo mappingInfo2 = mapping.get("T2"); - assertEquals("insert into TABLE2 (code, prop1, prop2) values(?, ?, ?)", + assertEquals("insert into table2 (code, prop1, prop2) values(?, ?, ?)", mappingInfo2.createInsertStatement()); assertEquals(2, mapping.size()); } @@ -258,16 +267,69 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase } } + @Test + public void testDatabaseMetaDataMissingCodeColumn() throws Exception + { + FileUtilities.writeToFile(mappingFile, "[T1:REPORT1,C]"); + try + { + materialReportingTask.setUp("", properties); + fail("EnvironmentFailureException expected"); + } catch (EnvironmentFailureException ex) + { + assertEquals("Missing column 'c' in table 'report1' of report database.", + ex.getMessage()); + } + } + + @Test + public void testDatabaseMetaDataMissingTable() throws Exception + { + FileUtilities.writeToFile(mappingFile, "[T1:REPORT3,C]"); + try + { + materialReportingTask.setUp("", properties); + fail("EnvironmentFailureException expected"); + } catch (EnvironmentFailureException ex) + { + assertEquals("Missing table 'report3' in report database.", ex.getMessage()); + } + } + + @Test + public void testDatabaseMetaDataCodeColumnOfWrongType() throws Exception + { + FileUtilities.writeToFile(mappingFile, "[T1:REPORT1,ID]"); + try + { + materialReportingTask.setUp("", properties); + fail("EnvironmentFailureException expected"); + } catch (EnvironmentFailureException ex) + { + assertEquals("Column 'id' of table 'report1' is not of type VARCHAR.", ex.getMessage()); + } + } + + @Test + public void testDatabaseMetaDataMissingPropertyColumn() throws Exception + { + FileUtilities.writeToFile(mappingFile, "[T1:REPORT2,CODE]\nP1:my_prop"); + try + { + materialReportingTask.setUp("", properties); + fail("EnvironmentFailureException expected"); + } catch (EnvironmentFailureException ex) + { + assertEquals("Missing column 'my_prop' in table 'report2' of report database.", + ex.getMessage()); + } + } + @Test public void test() throws Exception { FileUtilities.writeToFile(mappingFile, "# my mapping\n[T1:REPORT1,CODE]\n\n" + "[T2: REPORT2, code]\nP2: prop2\nP1:prop1"); - Properties properties = new Properties(); - properties.setProperty("database-driver", "org.postgresql.Driver"); - properties.setProperty("database-url", "jdbc:postgresql://localhost/" + databaseName); - properties.setProperty("database-username", "postgres"); - properties.setProperty(MaterialReportingTask.MAPPING_FILE_KEY, mappingFile.getPath()); materialReportingTask.setUp("", properties); final Material m1 = new MaterialBuilder().code("M1").type("T1").property("P1", "42").getMaterial(); @@ -295,7 +357,7 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase List<?> result = new JdbcTemplate(dbConfigContext.getDataSource()).query("select * from report1", new ColumnMapRowMapper()); - assertEquals("[{code=M1, description=null}]", result.toString()); + assertEquals("[{id=null, code=M1, description=null}]", result.toString()); result = new JdbcTemplate(dbConfigContext.getDataSource()).query( "select * from report2 order by code", new ColumnMapRowMapper()); -- GitLab