diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTask.java
index 92dfc96cc0f28c99a39a77c7a978ac83eeff768d..e079732f461ec994fb6030710ae1ee33436a521f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTask.java
@@ -151,8 +151,8 @@ public class DataSetAndPathInfoDBConsistencyCheckTask extends AbstractMaintenanc
executeInterruptingMode();
} else
{
- Date youngerThanDate = new Date(timeProvider.getTimeInMilliseconds() - timeInterval);
- List<DataSet> dataSets = getNextDataSets(youngerThanDate, null);
+ Date youngerThanDate = getOriginalStartingTime();
+ List<DataSet> dataSets = getNextDataSets(youngerThanDate, null, readTimestampAndCodeFromStateFile());
operationLog.info("Check " + dataSets.size() + " data sets registered since "
+ DATE_FORMAT.format(youngerThanDate));
DataSetAndPathInfoDBConsistencyChecker checker =
@@ -197,9 +197,10 @@ public class DataSetAndPathInfoDBConsistencyCheckTask extends AbstractMaintenanc
}
DataSetAndPathInfoDBConsistencyChecker checker =
new DataSetAndPathInfoDBConsistencyChecker(fileProvider, pathInfoProvider);
- Date lastRegistrationDate = getLastRegistrationDate();
+ Date originalStartingTime = getOriginalStartingTime();
+ Date lastRegistrationDate = getLastRegistrationDate(originalStartingTime);
Date registrationDate = lastRegistrationDate;
- DataSetIterable dataSetIterable = new DataSetIterable(registrationDate);
+ DataSetIterable dataSetIterable = new DataSetIterable(registrationDate, originalStartingTime);
for (DataSet dataSet : dataSetIterable)
{
checker.checkDataSet(dataSet.getCode());
@@ -216,10 +217,12 @@ public class DataSetAndPathInfoDBConsistencyCheckTask extends AbstractMaintenanc
private class DataSetIterable implements Iterable<DataSet>
{
private Date lastRegistrationDate;
+ private Date originalStartingTime;
- DataSetIterable(Date lastRegistrationDate)
+ DataSetIterable(Date lastRegistrationDate, Date originalStartingTime)
{
this.lastRegistrationDate = lastRegistrationDate;
+ this.originalStartingTime = originalStartingTime;
}
@Override
@@ -235,7 +238,13 @@ public class DataSetAndPathInfoDBConsistencyCheckTask extends AbstractMaintenanc
{
if (currentChunk == null || index == currentChunk.size())
{
- currentChunk = getNextDataSets(lastRegistrationDate, chunkSize);
+ String timestampAndCode = readTimestampAndCodeFromStateFile();
+ currentChunk = getNextDataSets(lastRegistrationDate, chunkSize, timestampAndCode);
+ if (currentChunk.isEmpty())
+ {
+ lastRegistrationDate = originalStartingTime;
+ currentChunk = getNextDataSets(lastRegistrationDate, chunkSize, null);
+ }
operationLog.info("Check " + currentChunk.size() + " data sets registered since "
+ DATE_FORMAT.format(lastRegistrationDate));
index = 0;
@@ -260,10 +269,9 @@ public class DataSetAndPathInfoDBConsistencyCheckTask extends AbstractMaintenanc
return interval.isInTimeInterval(new Date(timeProvider.getTimeInMilliseconds()));
}
- private List<DataSet> getNextDataSets(Date registrationDate, Integer count)
+ private List<DataSet> getNextDataSets(Date registrationDate, Integer count, String timestampAndCodeOrNull)
{
String sessionToken = login();
- String timestampAndCodeOrNull = readTimestampAndCodeFromStateFile();
DataSetSearchCriteria searchCriteria = new DataSetSearchCriteria();
searchCriteria.withRegistrationDate().thatIsLaterThanOrEqualTo(registrationDate);
PhysicalDataSearchCriteria physicalDataSearchCriteria = searchCriteria.withPhysicalData();
@@ -301,7 +309,7 @@ public class DataSetAndPathInfoDBConsistencyCheckTask extends AbstractMaintenanc
return timeStampAndCodeOfDataSet.compareTo(timestampAndCodeOrNull) > 0;
}
- private Date getLastRegistrationDate()
+ private Date getLastRegistrationDate(Date originalStartingTime)
{
if (stateFile.exists())
{
@@ -316,6 +324,11 @@ public class DataSetAndPathInfoDBConsistencyCheckTask extends AbstractMaintenanc
+ stateFile.getAbsolutePath() + ", timestamp: " + timestamp);
}
}
+ return originalStartingTime;
+ }
+
+ private Date getOriginalStartingTime()
+ {
return new Date(timeProvider.getTimeInMilliseconds() - timeInterval);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DataSetAndPathInfoDBConsistencyChecker.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DataSetAndPathInfoDBConsistencyChecker.java
index 4dcd9adab173ccb13bcb863549c66152c4101d68..78222e9f1ac59c60158aada251cbcfef140bcbb4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DataSetAndPathInfoDBConsistencyChecker.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DataSetAndPathInfoDBConsistencyChecker.java
@@ -29,6 +29,7 @@ import java.util.TreeMap;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.collection.CollectionUtils;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.io.IOUtilities;
import ch.systemsx.cisd.common.logging.LogCategory;
@@ -159,7 +160,7 @@ public class DataSetAndPathInfoDBConsistencyChecker
{
StringBuilder builder = new StringBuilder();
builder.append("Data sets checked:\n\n");
- builder.append(String.join(", ", dataSets));
+ builder.append(CollectionUtils.abbreviate(dataSets, 1000));
builder.append("\n\n");
builder.append("Differences found:\n\n");
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTaskTest.java
index c9f9445860ecda6de56e7e1a88e62995169d2795..4ee80b8793f0b6717400af2d9a69f7f50ba49366 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/path/DataSetAndPathInfoDBConsistencyCheckTaskTest.java
@@ -233,7 +233,7 @@ public class DataSetAndPathInfoDBConsistencyCheckTaskTest extends AbstractFileSy
logRecorder.getLogLines(),
hasItem("ERROR NOTIFY.DataSetAndPathInfoDBConsistencyCheckTask - File system and path info DB consistency check report for all data sets since 1977-06-15 23:53:08"));
assertThat(logRecorder.getLogLines(), hasItem("Data sets checked:"));
- assertThat(logRecorder.getLogLines(), hasItem("ds1, ds2, ds3, ds4, ds5"));
+ assertThat(logRecorder.getLogLines(), hasItem("[ds1, ds2, ds3, ds4, ds5]"));
assertThat(logRecorder.getLogLines(), hasItem("Differences found:"));
assertThat(logRecorder.getLogLines(), hasItem("Data set ds1:"));
assertThat(logRecorder.getLogLines(),
@@ -289,6 +289,7 @@ public class DataSetAndPathInfoDBConsistencyCheckTaskTest extends AbstractFileSy
RecordingMatcher<DataSetSearchCriteria> searchCriteriaRecorder1 = prepareListDataSets(ds1, ds2);
RecordingMatcher<DataSetSearchCriteria> searchCriteriaRecorder2 = prepareListDataSets(ds5, ds4, ds3);
RecordingMatcher<DataSetSearchCriteria> searchCriteriaRecorder3 = prepareListDataSets(ds5, ds4, ds3);
+ RecordingMatcher<DataSetSearchCriteria> searchCriteriaRecorder4 = prepareListDataSets();
MockContent fc1 = prepareContentProvider(fileProvider, "ds1", ":0:0", "a:34:9");
MockContent pic1 = prepareContentProvider(pathInfoProvider, "ds1", ":0:0", "a:34:9");
MockContent fc2 = prepareContentProvider(fileProvider, "ds2", ":0:0", "b:35:10");
@@ -322,18 +323,24 @@ public class DataSetAndPathInfoDBConsistencyCheckTaskTest extends AbstractFileSy
+ " with attribute 'storageConfirmation' true\n"
+ " with attribute 'status' AVAILABLE\n", searchCriteriaRecorder2.recordedObject().toString());
assertThat(logRecorder.getLogLines(), hasItem("INFO OPERATION.DataSetAndPathInfoDBConsistencyCheckTask - "
- + "Check 0 data sets registered since 1977-06-15 23:53:05"));
+ + "Check 0 data sets registered since 1977-06-15 23:52:55"));
assertEquals("DATASET\n"
+ " with operator 'AND'\n"
+ " with attribute 'registration_date' later than or equal to 'Wed Jun 15 23:53:05 CET 1977'\n"
+ " with physical_data:\n"
+ " with attribute 'storageConfirmation' true\n"
+ " with attribute 'status' AVAILABLE\n", searchCriteriaRecorder3.recordedObject().toString());
+ assertEquals("DATASET\n"
+ + " with operator 'AND'\n"
+ + " with attribute 'registration_date' later than or equal to 'Wed Jun 15 23:52:55 CET 1977'\n"
+ + " with physical_data:\n"
+ + " with attribute 'storageConfirmation' true\n"
+ + " with attribute 'status' AVAILABLE\n", searchCriteriaRecorder4.recordedObject().toString());
assertThat(logRecorder.getLogLines(), hasItem("ERROR NOTIFY.DataSetAndPathInfoDBConsistencyCheckTask - "
+ "File system and path info DB consistency check report for all data sets "
+ "between 1977-06-15 23:52:55 and 1977-06-15 23:53:05"));
assertThat(logRecorder.getLogLines(), hasItem("Data sets checked:"));
- assertThat(logRecorder.getLogLines(), hasItem("ds1, ds2, ds3, ds4, ds5"));
+ assertThat(logRecorder.getLogLines(), hasItem("[ds1, ds2, ds3, ds4, ds5]"));
assertThat(logRecorder.getLogLines(), hasItem("Differences found:"));
assertThat(logRecorder.getLogLines(), hasItem("Data set ds4:"));
assertThat(logRecorder.getLogLines(), hasItem("- 'd' CRC32 checksum in the file system = 00000123 but in the path info database = 00000012"));