diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask.java new file mode 100644 index 0000000000000000000000000000000000000000..e83c7144e244993700affaaf0baa7884b45fefe4 --- /dev/null +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/task/DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask.java @@ -0,0 +1,206 @@ +/* + * Copyright 2013 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.generic.server.task; + +import java.io.File; +import java.text.ParseException; +import java.util.Arrays; +import java.util.Collection; +import java.util.Date; +import java.util.HashSet; +import java.util.List; +import java.util.Properties; + +import org.apache.commons.lang.time.DateFormatUtils; +import org.apache.commons.lang.time.DateUtils; +import org.apache.log4j.Logger; + +import ch.systemsx.cisd.common.filesystem.FileUtilities; +import ch.systemsx.cisd.common.logging.LogCategory; +import ch.systemsx.cisd.common.logging.LogFactory; +import ch.systemsx.cisd.common.maintenance.IMaintenanceTask; +import ch.systemsx.cisd.common.properties.PropertyUtils; +import ch.systemsx.cisd.common.utilities.ITimeProvider; +import ch.systemsx.cisd.common.utilities.SystemTimeProvider; +import ch.systemsx.cisd.openbis.generic.server.CommonServiceProvider; +import ch.systemsx.cisd.openbis.generic.server.ICommonServerForInternalUse; +import ch.systemsx.cisd.openbis.generic.server.dataaccess.DynamicPropertyEvaluationOperation; +import ch.systemsx.cisd.openbis.generic.server.dataaccess.IDynamicPropertyEvaluationScheduler; +import ch.systemsx.cisd.openbis.generic.server.dataaccess.IDynamicPropertyEvaluationSchedulerWithQueue; +import ch.systemsx.cisd.openbis.generic.shared.basic.TechId; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.CompareType; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchCriteria; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchCriterion; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchField; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Material; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.MaterialAttributeSearchFieldKind; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SearchCriteriaConnection; +import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePE; +import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO; +import ch.systemsx.cisd.openbis.generic.shared.util.SimplePropertyValidator.SupportedDatePattern; + +/** + * Maintenance task for re-evaluation of dynamic properties of entities which have material properties which have changed since the last run of the + * task. This is also true if the material properties of the material properties have changed. + * <p> + * Currently only samples are supported. + * + * @author Franz-Josef Elmer + */ +public class DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask implements IMaintenanceTask +{ + private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, + DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask.class); + + static final String TIMESTAMP_FILE_KEY = "timestamp-file"; + + static final String DEFAULT_TIMESTAMP_FILE = "../../../data/" + + DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask.class.getSimpleName() + "-timestamp.txt"; + + static final String CANCEL_IF_NO_TIMESTAMP_FILE_KEY = "cancel-if-no-timestamp-file"; + + static final String INITIAL_TIMESTAMP_KEY = "initial-timestamp"; + + private final ICommonServerForInternalUse server; + + private final IDynamicPropertyEvaluationScheduler scheduler; + + private final ITimeProvider timeProvider; + + private File timestampFile; + + private String initialTimestamp; + + public DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask() + { + this(CommonServiceProvider.getCommonServer(), CommonServiceProvider.getDAOFactory().getPersistencyResources() + .getDynamicPropertyEvaluationScheduler(), SystemTimeProvider.SYSTEM_TIME_PROVIDER); + } + + DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask(ICommonServerForInternalUse server, + IDynamicPropertyEvaluationScheduler scheduler, ITimeProvider timeProvider) + { + this.server = server; + this.scheduler = scheduler; + this.timeProvider = timeProvider; + } + + @Override + public void setUp(String pluginName, Properties properties) + { + timestampFile = new File(properties.getProperty(TIMESTAMP_FILE_KEY, DEFAULT_TIMESTAMP_FILE)); + initialTimestamp = PropertyUtils.getMandatoryProperty(properties, INITIAL_TIMESTAMP_KEY); + } + + @Override + public void execute() + { + String timestamp = getTimestamp(); + SessionContextDTO contextOrNull = server.tryToAuthenticateAsSystem(); + if (contextOrNull == null) + { + operationLog.warn("Couldn't authenticate as system."); + return; + } + String sessionToken = contextOrNull.getSessionToken(); + try + { + String newTimestamp = createTimestamp(); + Collection<TechId> allMaterialIds = getAllChangedMaterials(sessionToken, timestamp); + if (allMaterialIds.isEmpty() == false) + { + List<Sample> samples = server.listSamplesByMaterialProperties(sessionToken, allMaterialIds); + operationLog.info(samples.size() + " samples found for changed materials."); + if (samples.isEmpty() == false) + { + List<Long> ids = TechId.asLongs(TechId.createList(samples)); + DynamicPropertyEvaluationOperation operation = DynamicPropertyEvaluationOperation.evaluate(SamplePE.class, ids); + scheduler.scheduleUpdate(operation); + if (scheduler instanceof IDynamicPropertyEvaluationSchedulerWithQueue) + { + ((IDynamicPropertyEvaluationSchedulerWithQueue) scheduler).synchronizeThreadQueue(); + } + } + } + saveTimestamp(newTimestamp); + } finally + { + server.logout(sessionToken); + } + } + + private Collection<TechId> getAllChangedMaterials(String sessionToken, String timestamp) + { + DetailedSearchCriteria criteria = new DetailedSearchCriteria(); + DetailedSearchCriterion criterion = + new DetailedSearchCriterion( + DetailedSearchField + .createAttributeField(MaterialAttributeSearchFieldKind.MODIFICATION_DATE), + CompareType.MORE_THAN_OR_EQUAL, timestamp); + criteria.setCriteria(Arrays.asList(criterion)); + criteria.setConnection(SearchCriteriaConnection.MATCH_ALL); + List<Material> materials = server.searchForMaterials(sessionToken, criteria); + int numberOfChangedMaterials = materials.size(); + operationLog.info(numberOfChangedMaterials + " materials changed since [" + timestamp + "]."); + Collection<TechId> allMaterialIds = new HashSet<TechId>(); + Collection<TechId> materialIds = TechId.createList(materials); + while (materialIds.isEmpty() == false) + { + materialIds.removeAll(allMaterialIds); + allMaterialIds.addAll(materialIds); + materialIds = server.listMaterialIdsByMaterialProperties(sessionToken, materialIds); + } + if (numberOfChangedMaterials != allMaterialIds.size()) + { + operationLog.info(allMaterialIds.size() + " materials in total changed."); + } + return allMaterialIds; + } + + private String getTimestamp() + { + String timestamp = null; + if (timestampFile.isFile()) + { + timestamp = FileUtilities.loadToString(timestampFile).trim(); + try + { + DateUtils.parseDate(timestamp, new String[] { SupportedDatePattern.CANONICAL_DATE_PATTERN.getPattern() }); + } catch (ParseException ex) + { + operationLog.warn("Invalid timestamp in file '" + timestampFile + "': " + timestamp); + timestamp = null; + } + } + return timestamp != null ? timestamp : initialTimestamp; + } + + private String createTimestamp() + { + return DateFormatUtils.format(new Date(timeProvider.getTimeInMilliseconds()), + SupportedDatePattern.CANONICAL_DATE_PATTERN.getPattern()); + } + + private void saveTimestamp(String newTimestamp) + { + timestampFile.getParentFile().mkdirs(); + FileUtilities.writeToFile(timestampFile, newTimestamp); + operationLog.info("Timestamp [" + newTimestamp + "] saved in '" + timestampFile + "'."); + } + +} diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTaskTest.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTaskTest.java new file mode 100644 index 0000000000000000000000000000000000000000..bc5153825de1e7dbcfe1e66be1dd5a93b57de5c6 --- /dev/null +++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/task/DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTaskTest.java @@ -0,0 +1,274 @@ +/* + * Copyright 2013 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.generic.server.task; + +import static ch.systemsx.cisd.openbis.generic.server.task.DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask.INITIAL_TIMESTAMP_KEY; +import static ch.systemsx.cisd.openbis.generic.server.task.DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask.TIMESTAMP_FILE_KEY; + +import java.io.File; +import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.List; +import java.util.Properties; + +import org.apache.log4j.Level; +import org.jmock.Expectations; +import org.jmock.Mockery; +import org.testng.annotations.AfterMethod; +import org.testng.annotations.BeforeMethod; +import org.testng.annotations.Test; + +import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase; +import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; +import ch.systemsx.cisd.common.filesystem.FileUtilities; +import ch.systemsx.cisd.common.logging.BufferedAppender; +import ch.systemsx.cisd.common.test.RecordingMatcher; +import ch.systemsx.cisd.common.utilities.ITimeProvider; +import ch.systemsx.cisd.common.utilities.MockTimeProvider; +import ch.systemsx.cisd.openbis.generic.server.ICommonServerForInternalUse; +import ch.systemsx.cisd.openbis.generic.server.dataaccess.DynamicPropertyEvaluationOperation; +import ch.systemsx.cisd.openbis.generic.server.dataaccess.IDynamicPropertyEvaluationSchedulerWithQueue; +import ch.systemsx.cisd.openbis.generic.shared.basic.TechId; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DetailedSearchCriteria; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Material; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.MaterialBuilder; +import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.SampleBuilder; +import ch.systemsx.cisd.openbis.generic.shared.dto.SessionContextDTO; + +/** + * @author Franz-Josef Elmer + */ +public class DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTaskTest extends AbstractFileSystemTestCase +{ + private static final String SESSION_TOKEN = "my-session"; + + private BufferedAppender logRecorder; + + private Mockery context; + + private ICommonServerForInternalUse server; + + private IDynamicPropertyEvaluationSchedulerWithQueue scheduler; + + private ITimeProvider timeProvider; + + private DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask task; + + private Properties properties; + + private File timestampFile; + + @BeforeMethod + public void setUpMocksAndProperties() + { + logRecorder = new BufferedAppender("%-5p %m%n", Level.DEBUG); + context = new Mockery(); + server = context.mock(ICommonServerForInternalUse.class); + scheduler = context.mock(IDynamicPropertyEvaluationSchedulerWithQueue.class); + timeProvider = new MockTimeProvider(7000, 1000); + task = new DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask(server, scheduler, timeProvider); + properties = new Properties(); + timestampFile = new File(workingDirectory, "timestamp.txt"); + properties.setProperty(TIMESTAMP_FILE_KEY, timestampFile.getPath()); + } + + @AfterMethod + public void tearDown() throws Exception + { + logRecorder.reset(); + context.assertIsSatisfied(); + } + + @Test + public void testMissingInitialTimestamp() + { + try + { + task.setUp("test", properties); + fail("ConfigurationFailureException expected."); + } catch (ConfigurationFailureException ex) + { + assertEquals("Given key 'initial-timestamp' not found in properties '[timestamp-file]'", ex.getMessage()); + } + + context.assertIsSatisfied(); + } + + @Test + public void testAuthenticationAsSystemFailed() + { + properties.setProperty(DynamicPropertyEvaluationTriggeredByMaterialChangeMaintenanceTask.INITIAL_TIMESTAMP_KEY, "2011-01-01"); + task.setUp("test", properties); + prepareLogInAndOut(null); + + task.execute(); + + assertEquals("WARN Couldn't authenticate as system.", logRecorder.getLogContent()); + context.assertIsSatisfied(); + } + + @Test + public void testExecuteForFoundSamplesSinceInitialTimestamp() + { + properties.setProperty(INITIAL_TIMESTAMP_KEY, "2011-01-01"); + task.setUp("test", properties); + prepareLogInAndOut(SESSION_TOKEN); + Material m1 = new MaterialBuilder().id(101L).code("M101").type("T").getMaterial(); + RecordingMatcher<DetailedSearchCriteria> criteriaRecorder = prepareSearchForMaterials(m1); + prepareListMaterialIds(Arrays.asList(new TechId(101L)), 102L, 103L); + prepareListMaterialIds(Arrays.asList(new TechId(102L), new TechId(103L))); + Sample s1 = new SampleBuilder("/S/1").id(1).getSample(); + Sample s2 = new SampleBuilder("/S/2").id(2).getSample(); + RecordingMatcher<Collection<TechId>> materialIdsRecoder = prepareListSamples(s1, s2); + RecordingMatcher<DynamicPropertyEvaluationOperation> scheduledOperationsRecorder = + prepareScheduleDynamicPropertyEvaluation(); + + task.execute(); + + assertEquals("ATTRIBUTE MODIFICATION_DATE: 2011-01-01 (without wildcards)", + criteriaRecorder.recordedObject().toString()); + List<Long> materialIds = TechId.asLongs(materialIdsRecoder.recordedObject()); + Collections.sort(materialIds); + assertEquals("[101, 102, 103]", materialIds.toString()); + assertEquals("ch.systemsx.cisd.openbis.generic.shared.dto.SamplePE: [1, 2]", + scheduledOperationsRecorder.recordedObject().toString()); + assertEquals("INFO 1 materials changed since [2011-01-01].\n" + + "INFO 3 materials in total changed.\n" + + "INFO 2 samples found for changed materials.\n" + + "INFO Timestamp [1970-01-01 01:00:07 +0100] saved in '" + timestampFile + "'.", + logRecorder.getLogContent()); + assertEquals("1970-01-01 01:00:07 +0100", FileUtilities.loadToString(timestampFile).trim()); + context.assertIsSatisfied(); + } + + @Test + public void testExecuteForNoSamplesSinceTimestampFromFile() + { + FileUtilities.writeToFile(timestampFile, "2013-09-05 09:19:54 +0200"); + properties.setProperty(INITIAL_TIMESTAMP_KEY, "2011-01-01"); + task.setUp("test", properties); + prepareLogInAndOut(SESSION_TOKEN); + Material m1 = new MaterialBuilder().id(101L).code("M101").type("T").getMaterial(); + RecordingMatcher<DetailedSearchCriteria> criteriaRecorder = prepareSearchForMaterials(m1); + prepareListMaterialIds(Arrays.asList(new TechId(101L))); + RecordingMatcher<Collection<TechId>> materialIdsRecoder = prepareListSamples(); + + task.execute(); + + assertEquals("ATTRIBUTE MODIFICATION_DATE: 2013-09-05 09:19:54 +0200 (without wildcards)", + criteriaRecorder.recordedObject().toString()); + List<Long> materialIds = TechId.asLongs(materialIdsRecoder.recordedObject()); + Collections.sort(materialIds); + assertEquals("[101]", materialIds.toString()); + assertEquals("INFO 1 materials changed since [2013-09-05 09:19:54 +0200].\n" + + "INFO 0 samples found for changed materials.\n" + + "INFO Timestamp [1970-01-01 01:00:07 +0100] saved in '" + timestampFile + "'.", + logRecorder.getLogContent()); + assertEquals("1970-01-01 01:00:07 +0100", FileUtilities.loadToString(timestampFile).trim()); + context.assertIsSatisfied(); + } + + @Test + public void testExecuteForNoMaterialChangedSinceTimestampFromFile() + { + FileUtilities.writeToFile(timestampFile, "2013-09-05 09:19:54 +0200"); + properties.setProperty(INITIAL_TIMESTAMP_KEY, "2011-01-01"); + task.setUp("test", properties); + prepareLogInAndOut(SESSION_TOKEN); + RecordingMatcher<DetailedSearchCriteria> criteriaRecorder = prepareSearchForMaterials(); + + task.execute(); + + assertEquals("ATTRIBUTE MODIFICATION_DATE: 2013-09-05 09:19:54 +0200 (without wildcards)", + criteriaRecorder.recordedObject().toString()); + assertEquals("INFO 0 materials changed since [2013-09-05 09:19:54 +0200].\n" + + "INFO Timestamp [1970-01-01 01:00:07 +0100] saved in '" + timestampFile + "'.", + logRecorder.getLogContent()); + assertEquals("1970-01-01 01:00:07 +0100", FileUtilities.loadToString(timestampFile).trim()); + context.assertIsSatisfied(); + } + + private void prepareLogInAndOut(final String sessionTokenOrNull) + { + context.checking(new Expectations() + { + { + one(server).tryToAuthenticateAsSystem(); + SessionContextDTO sessionContext = new SessionContextDTO(); + sessionContext.setSessionToken(sessionTokenOrNull); + will(returnValue(sessionTokenOrNull == null ? null : sessionContext)); + + if (sessionTokenOrNull != null) + { + one(server).logout(sessionTokenOrNull); + } + } + }); + } + + private RecordingMatcher<DetailedSearchCriteria> prepareSearchForMaterials(final Material... materials) + { + final RecordingMatcher<DetailedSearchCriteria> criteriaRecorder = new RecordingMatcher<DetailedSearchCriteria>(); + context.checking(new Expectations() + { + { + one(server).searchForMaterials(with(SESSION_TOKEN), with(criteriaRecorder)); + will(returnValue(Arrays.asList(materials))); + } + }); + return criteriaRecorder; + } + + private void prepareListMaterialIds(final Collection<TechId> propertiesMaterialIds, final Long... materialIds) + { + context.checking(new Expectations() + { + { + one(server).listMaterialIdsByMaterialProperties(SESSION_TOKEN, propertiesMaterialIds); + will(returnValue(TechId.createList(Arrays.asList(materialIds)))); + } + }); + } + + private RecordingMatcher<Collection<TechId>> prepareListSamples(final Sample... samples) + { + final RecordingMatcher<Collection<TechId>> matcher = new RecordingMatcher<Collection<TechId>>(); + context.checking(new Expectations() + { + { + one(server).listSamplesByMaterialProperties(with(SESSION_TOKEN), with(matcher)); + will(returnValue(Arrays.asList(samples))); + } + }); + return matcher; + } + + private RecordingMatcher<DynamicPropertyEvaluationOperation> prepareScheduleDynamicPropertyEvaluation() + { + final RecordingMatcher<DynamicPropertyEvaluationOperation> matcher = new RecordingMatcher<DynamicPropertyEvaluationOperation>(); + context.checking(new Expectations() + { + { + one(scheduler).scheduleUpdate(with(matcher)); + one(scheduler).synchronizeThreadQueue(); + } + }); + return matcher; + } +} diff --git a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/builders/MaterialBuilder.java b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/builders/MaterialBuilder.java index ed4cbeae40dd87484924e51cb9c028a8bb75798f..486f9791d35a86beb85c431a5927eb6052eb87c1 100644 --- a/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/builders/MaterialBuilder.java +++ b/openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/shared/basic/dto/builders/MaterialBuilder.java @@ -42,6 +42,12 @@ public class MaterialBuilder return material; } + public MaterialBuilder id(Long id) + { + material.setId(id); + return this; + } + public MaterialBuilder code(String code) { material.setCode(code);