SP-39 reading and writing timestamp

SVN: 25276

openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTask.java

@@ -37,7 +37,9 @@ import java.util.Properties;
 import java.util.Set;
 import java.util.TreeMap;
 
+import org.apache.commons.lang.time.DateFormatUtils;
 import org.apache.log4j.Logger;
+import org.springframework.dao.IncorrectResultSizeDataAccessException;
 import org.springframework.jdbc.core.BatchPreparedStatementSetter;
 import org.springframework.jdbc.core.ColumnMapRowMapper;
 import org.springframework.jdbc.core.JdbcTemplate;
@@ -52,7 +54,9 @@ import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
+import ch.systemsx.cisd.common.utilities.ITimeProvider;
 import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.common.utilities.SystemTimeProvider;
 import ch.systemsx.cisd.dbmigration.SimpleDatabaseConfigurationContext;
 import ch.systemsx.cisd.openbis.generic.server.CommonServiceProvider;
 import ch.systemsx.cisd.openbis.generic.server.ICommonServerForInternalUse;
@@ -70,6 +74,7 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PropertyType;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SearchCriteriaConnection;
 import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO;
 import ch.systemsx.cisd.openbis.generic.shared.util.DataTypeUtils;
+import ch.systemsx.cisd.openbis.generic.shared.util.SimplePropertyValidator.SupportedDatePattern;
 
 /**
  * Task which feeds a reporting database with recently added/changed Materials.
@@ -78,6 +83,15 @@ import ch.systemsx.cisd.openbis.generic.shared.util.DataTypeUtils;
  */
 public class MaterialReportingTask implements IMaintenanceTask
 {
+    @Private
+    static final String READ_TIMESTAMP_SQL_KEY = "read-timestamp-sql";
+
+    @Private
+    static final String UPDATE_TIMESTAMP_SQL_KEY = "update-timestamp-sql";
+
+    @Private
+    static final String INSERT_TIMESTAMP_SQL_KEY = "insert-timestamp-sql";
+
     @Private
     static final String MAPPING_FILE_KEY = "mapping-file";
 
@@ -347,29 +361,38 @@ public class MaterialReportingTask implements IMaintenanceTask
 
     private final ICommonServerForInternalUse server;
 
+    private final ITimeProvider timeProvider;
+
     private Map<String, MappingInfo> mapping;
 
     private SimpleDatabaseConfigurationContext dbConfigurationContext;
 
     private JdbcTemplate jdbcTemplate;
 
+    private String readTimestampSql;
+
+    private String insertTimestampSql;
+
+    private String updateTimestampSql;
+
     public MaterialReportingTask()
     {
-        this(CommonServiceProvider.getCommonServer());
+        this(CommonServiceProvider.getCommonServer(), SystemTimeProvider.SYSTEM_TIME_PROVIDER);
     }
 
-    public MaterialReportingTask(ICommonServerForInternalUse server)
+    public MaterialReportingTask(ICommonServerForInternalUse server, ITimeProvider timeProvider)
     {
         this.server = server;
+        this.timeProvider = timeProvider;
     }
 
     public void setUp(String pluginName, Properties properties)
     {
         dbConfigurationContext = new SimpleDatabaseConfigurationContext(properties);
-        // String readTimestampSql = PropertyUtils.getMandatoryProperty(properties,
-        // "read-timestamp-sql");
-        // String writeTimestampSql = PropertyUtils.getMandatoryProperty(properties,
-        // "write-timestamp-sql");
+        readTimestampSql = PropertyUtils.getMandatoryProperty(properties, READ_TIMESTAMP_SQL_KEY);
+        updateTimestampSql =
+                PropertyUtils.getMandatoryProperty(properties, UPDATE_TIMESTAMP_SQL_KEY);
+        insertTimestampSql = properties.getProperty(INSERT_TIMESTAMP_SQL_KEY, updateTimestampSql);
         String mappingFileName = PropertyUtils.getMandatoryProperty(properties, MAPPING_FILE_KEY);
         mapping = readMappingFile(mappingFileName);
         Map<String, Map<String, PropertyType>> materialTypes = getMaterialTypes();
@@ -393,6 +416,7 @@ public class MaterialReportingTask implements IMaintenanceTask
             mappingInfo.injectDataTypeCodes(columns, propertyTypes);
         }
         jdbcTemplate = new JdbcTemplate(dbConfigurationContext.getDataSource());
+        checkTimestampReadingWriting();
     }
 
     public void execute()
@@ -415,6 +439,7 @@ public class MaterialReportingTask implements IMaintenanceTask
                 addOrUpdate(mappingInfo, materials);
             }
         }
+        writeTimestamp(new Date(timeProvider.getTimeInMilliseconds()));
         operationLog.info("Reporting finished.");
     }
 
@@ -551,7 +576,7 @@ public class MaterialReportingTask implements IMaintenanceTask
                 new DetailedSearchCriterion(
                         DetailedSearchField
                                 .createAttributeField(MaterialAttributeSearchFieldKind.MODIFICATION_DATE),
-                        CompareType.MORE_THAN_OR_EQUAL, readTimestamp(), "0");
+                        CompareType.MORE_THAN_OR_EQUAL, readTimestamp());
         criteria.setCriteria(Arrays.asList(criterion));
         criteria.setConnection(SearchCriteriaConnection.MATCH_ALL);
         List<Material> materials = server.searchForMaterials(sessionToken, criteria);
@@ -570,9 +595,59 @@ public class MaterialReportingTask implements IMaintenanceTask
         return result;
     }
 
+    private void checkTimestampReadingWriting()
+    {
+        Date timestamp;
+        try
+        {
+            timestamp = tryToReadTimestamp();
+        } catch (Exception ex)
+        {
+            throw new ConfigurationFailureException(
+                    "Couldn't get timestamp from report database. Property '"
+                            + READ_TIMESTAMP_SQL_KEY + "' could be invalid.", ex);
+        }
+        try
+        {
+            writeTimestamp(timestamp == null ? new Date(0) : timestamp);
+        } catch (Exception ex)
+        {
+            throw new ConfigurationFailureException(
+                    "Couldn't save timestamp to report database. Property '"
+                            + INSERT_TIMESTAMP_SQL_KEY + "' or '" + UPDATE_TIMESTAMP_SQL_KEY
+                            + "' could be invalid.", ex);
+        }
+    }
+
     private String readTimestamp()
     {
-        return "2012-02-20 10:33:44.6667";
+        Date timestamp = tryToReadTimestamp();
+        return timestamp == null ? "1970-01-01" : DateFormatUtils.format(timestamp,
+                SupportedDatePattern.CANONICAL_DATE_PATTERN.getPattern());
+    }
+
+    private Date tryToReadTimestamp()
+    {
+        try
+        {
+            return (Date) jdbcTemplate.queryForObject(readTimestampSql, Date.class);
+        } catch (IncorrectResultSizeDataAccessException ex)
+        {
+            int actualSize = ex.getActualSize();
+            if (actualSize == 0)
+            {
+                return null;
+            }
+            throw ex;
+        }
+    }
+
+    private void writeTimestamp(Date newTimestamp)
+    {
+        String sql = tryToReadTimestamp() == null ? insertTimestampSql : updateTimestampSql;
+        jdbcTemplate.update(sql, new Object[]
+            { newTimestamp }, new int[]
+            { Types.TIMESTAMP });
     }
 
     @Private

openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/MaterialReportingTaskTest.java

@@ -43,11 +43,15 @@ import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
 import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
 import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.common.test.RecordingMatcher;
+import ch.systemsx.cisd.common.utilities.ITimeProvider;
 import ch.systemsx.cisd.dbmigration.DatabaseConfigurationContext;
 import ch.systemsx.cisd.openbis.generic.server.ICommonServerForInternalUse;
 import ch.systemsx.cisd.openbis.generic.server.task.MaterialReportingTask.MappingInfo;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.CompareType;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataTypeCode;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchCriteria;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchCriterion;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchFieldKind;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Material;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialType;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.MaterialBuilder;
@@ -78,12 +82,15 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase
 
     private Properties properties;
 
+    private ITimeProvider timeProvider;
+
     @BeforeMethod
     public void setUpMocks() throws Exception
     {
         context = new Mockery();
         server = context.mock(ICommonServerForInternalUse.class);
-        materialReportingTask = new MaterialReportingTask(server);
+        timeProvider = context.mock(ITimeProvider.class);
+        materialReportingTask = new MaterialReportingTask(server, timeProvider);
 
         dbConfigContext = new DatabaseConfigurationContext();
         dbConfigContext.setDatabaseEngineCode("postgresql");
@@ -94,12 +101,19 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase
         dropTestDatabase();
         createTestDatabase();
         createTables(
+                "create table timestamp (timestamp timestamp)",
                 "create table report1 (id bigint, code varchar(20), description varchar(200))",
                 "create table report2 (code varchar(20), rank integer, greetings varchar(200), "
                         + "size double precision, organism varchar(100), material varchar(30), timestamp timestamp)");
         dbConfigContext.closeConnections();
         mappingFile = new File(workingDirectory, "mapping-file.txt");
         properties = new Properties();
+        properties.setProperty(MaterialReportingTask.READ_TIMESTAMP_SQL_KEY,
+                "select timestamp from timestamp");
+        properties.setProperty(MaterialReportingTask.INSERT_TIMESTAMP_SQL_KEY,
+                "insert into timestamp values(?)");
+        properties.setProperty(MaterialReportingTask.UPDATE_TIMESTAMP_SQL_KEY,
+                "update timestamp set timestamp = ?");
         properties.setProperty("database-driver", "org.postgresql.Driver");
         properties.setProperty("database-url", "jdbc:postgresql://localhost/" + databaseName);
         properties.setProperty("database-username", "postgres");
@@ -509,6 +523,9 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase
 
                     one(server).searchForMaterials(with(SESSION_TOKEN), with(criteriaRecorder));
                     will(returnValue(Arrays.asList(m1, m2, m3)));
+
+                    one(timeProvider).getTimeInMilliseconds();
+                    will(returnValue(24L * 3600L * 1000L * 60L));
                 }
             });
 
@@ -521,6 +538,15 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase
                 + "size=1.00000005E7, timestamp=1970-02-03 01:00:00.0}, "
                 + "{code=M3, greetings=hello, material=null, organism=null, rank=null, "
                 + "size=null, timestamp=null}]", result.toString());
+        assertEquals("[{timestamp=1970-03-02 01:00:00.0}]", loadTable("timestamp", false)
+                .toString());
+        DetailedSearchCriterion detailedSearchCriterion =
+                criteriaRecorder.recordedObject().getCriteria().get(0);
+        assertEquals(CompareType.MORE_THAN_OR_EQUAL, detailedSearchCriterion.getType());
+        assertEquals("MODIFICATION_DATE", detailedSearchCriterion.getField().getAttributeCode());
+        assertEquals(DetailedSearchFieldKind.ATTRIBUTE, detailedSearchCriterion.getField()
+                .getKind());
+        assertEquals("1970-01-01 01:00:00 +0100", detailedSearchCriterion.getValue());
         context.assertIsSatisfied();
     }
 
@@ -549,7 +575,8 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase
                 new MaterialBuilder().code("M4").type("T2").property("P2", "hi").getMaterial();
         final RecordingMatcher<DetailedSearchCriteria> criteriaRecorder =
                 new RecordingMatcher<DetailedSearchCriteria>();
-        final Sequence sequence = context.sequence("materials");
+        final Sequence searchMaterialSequence = context.sequence("materials");
+        final Sequence timeSequence = context.sequence("time");
         context.checking(new Expectations()
             {
                 {
@@ -560,11 +587,19 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase
 
                     one(server).searchForMaterials(with(SESSION_TOKEN), with(criteriaRecorder));
                     will(returnValue(Arrays.asList(m1, m2, m3)));
-                    inSequence(sequence);
+                    inSequence(searchMaterialSequence);
 
                     one(server).searchForMaterials(with(SESSION_TOKEN), with(criteriaRecorder));
                     will(returnValue(Arrays.asList(m1, m2v2, m4)));
-                    inSequence(sequence);
+                    inSequence(searchMaterialSequence);
+
+                    one(timeProvider).getTimeInMilliseconds();
+                    will(returnValue(24L * 3600L * 1000L * 60L));
+                    inSequence(timeSequence);
+
+                    one(timeProvider).getTimeInMilliseconds();
+                    will(returnValue(24L * 3600L * 1000L * 62L));
+                    inSequence(timeSequence);
                 }
             });
 
@@ -580,6 +615,12 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase
                 + "rank=null, size=null, timestamp=null}, "
                 + "{code=M4, greetings=hi, material=null, organism=null, "
                 + "rank=null, size=null, timestamp=null}]", result.toString());
+        assertEquals("[{timestamp=1970-03-04 01:00:00.0}]", loadTable("timestamp", false)
+                .toString());
+        assertEquals("1970-01-01 01:00:00 +0100", criteriaRecorder.getRecordedObjects().get(0)
+                .getCriteria().get(0).getValue());
+        assertEquals("1970-03-02 01:00:00 +0100", criteriaRecorder.getRecordedObjects().get(1)
+                .getCriteria().get(0).getValue());
         context.assertIsSatisfied();
     }
 
@@ -614,9 +655,14 @@ public class MaterialReportingTaskTest extends AbstractFileSystemTestCase
     }
 
     private List<?> loadTable(String tableName)
+    {
+        return loadTable(tableName, true);
+    }
+
+    private List<?> loadTable(String tableName, boolean orderByCode)
     {
         return new JdbcTemplate(dbConfigContext.getDataSource()).query("select * from " + tableName
-                + " order by code", new ColumnMapRowMapper()
+                + (orderByCode ? " order by code" : ""), new ColumnMapRowMapper()
             {
                 @SuppressWarnings("rawtypes")
                 @Override