LMS-791 index datasets when experiment properties are changed + unit tests + updated test db index

SVN: 10326

openbis/source/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/search/LuceneQueryBuilder.java

@@ -20,6 +20,7 @@ import static ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldC
 import static ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldConstants.PREFIX_EXPERIMENT;
 import static ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldConstants.PREFIX_EXPERIMENT_TYPE;
 import static ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldConstants.PREFIX_GROUP;
+import static ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldConstants.PREFIX_PROCEDURE;
 import static ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldConstants.PREFIX_PROJECT;
 import static ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldConstants.PREFIX_SAMPLE;
 import static ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldConstants.PREFIX_SAMPLE_TYPE;
@@ -56,7 +57,8 @@ public class LuceneQueryBuilder
             LogFactory.getLogger(LogCategory.OPERATION, LuceneQueryBuilder.class);
 
     /** @throws UserFailureException when some search patterns are incorrect */
-    public static Query createQuery(DataSetSearchCriteria dataSetCriteria) throws UserFailureException
+    public static Query createQuery(DataSetSearchCriteria dataSetCriteria)
+            throws UserFailureException
     {
         List<DataSetSearchCriterion> criteria = dataSetCriteria.getCriteria();
         Occur occureCondition = createOccureCondition(dataSetCriteria.getConnection());
@@ -162,6 +164,7 @@ public class LuceneQueryBuilder
     private static String tryGetIndexFieldName(DataSetSearchField searchField)
     {
         DataSetSearchFieldKind fieldKind = searchField.getKind();
+        String experimentField = PREFIX_PROCEDURE + PREFIX_EXPERIMENT;
         switch (fieldKind)
         {
             case DATA_SET_CODE:
@@ -171,15 +174,15 @@ public class LuceneQueryBuilder
             case FILE_TYPE:
                 return SearchFieldConstants.PREFIX_FILE_FORMAT_TYPE + CODE;
             case GROUP:
-                return PREFIX_EXPERIMENT + PREFIX_PROJECT + PREFIX_GROUP + CODE;
+                return experimentField + PREFIX_PROJECT + PREFIX_GROUP + CODE;
             case PROJECT:
-                return PREFIX_EXPERIMENT + PREFIX_PROJECT + CODE;
+                return experimentField + PREFIX_PROJECT + CODE;
             case EXPERIMENT:
-                return PREFIX_EXPERIMENT + CODE;
+                return experimentField + CODE;
             case EXPERIMENT_TYPE:
-                return PREFIX_EXPERIMENT + PREFIX_EXPERIMENT_TYPE + CODE;
+                return experimentField + PREFIX_EXPERIMENT_TYPE + CODE;
             case EXPERIMENT_PROPERTY:
-                return PREFIX_EXPERIMENT + getPropertyIndexField(searchField.getPropertyCode());
+                return experimentField + getPropertyIndexField(searchField.getPropertyCode());
             case SAMPLE:
                 return PREFIX_SAMPLE + CODE;
             case SAMPLE_TYPE:

openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/AttachmentPE.java

@@ -38,6 +38,7 @@ import org.apache.commons.lang.builder.ToStringBuilder;
 import org.apache.lucene.document.Document;
 import org.apache.lucene.document.Field;
 import org.hibernate.search.annotations.ClassBridge;
+import org.hibernate.search.annotations.ContainedIn;
 import org.hibernate.search.annotations.Index;
 import org.hibernate.search.annotations.Store;
 import org.hibernate.search.bridge.FieldBridge;
@@ -169,6 +170,7 @@ public class AttachmentPE extends HibernateAbstractRegistrationHolder implements
     @JoinColumn(name = ColumnNames.EXPERIMENT_COLUMN, updatable = false)
     @Private
     // for Hibernate and bean conversion only
+    @ContainedIn
     public ExperimentPE getParentInternal()
     {
         return parent;

openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/ExperimentPE.java

@@ -48,6 +48,7 @@ import org.hibernate.annotations.CacheConcurrencyStrategy;
 import org.hibernate.annotations.Cascade;
 import org.hibernate.annotations.Generated;
 import org.hibernate.annotations.GenerationTime;
+import org.hibernate.search.annotations.ContainedIn;
 import org.hibernate.search.annotations.DocumentId;
 import org.hibernate.search.annotations.Field;
 import org.hibernate.search.annotations.Index;
@@ -144,8 +145,6 @@ public class ExperimentPE implements IEntityPropertiesHolder<ExperimentPropertyP
 
     private Date modificationDate;
 
-    // private Date modificationDate;
-
     @Column(name = ColumnNames.REGISTRATION_TIMESTAMP_COLUMN, nullable = false, insertable = false, updatable = false)
     @Generated(GenerationTime.INSERT)
     public Date getRegistrationDate()
@@ -361,7 +360,8 @@ public class ExperimentPE implements IEntityPropertiesHolder<ExperimentPropertyP
     }
 
     @OneToMany(fetch = FetchType.LAZY, mappedBy = "experimentInternal")
-    @JoinColumn(name = ColumnNames.EXPERIMENT_COLUMN, updatable = false)
+    @JoinColumn(name = ColumnNames.EXPERIMENT_COLUMN, updatable = true)
+    @ContainedIn
     private List<ProcedurePE> getExperimentProcedures()
     {
         return procedures;

openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/ProcedurePE.java

@@ -39,6 +39,9 @@ import org.apache.commons.lang.builder.HashCodeBuilder;
 import org.apache.commons.lang.builder.ToStringBuilder;
 import org.hibernate.annotations.Generated;
 import org.hibernate.annotations.GenerationTime;
+import org.hibernate.search.annotations.ContainedIn;
+import org.hibernate.search.annotations.DocumentId;
+import org.hibernate.search.annotations.Indexed;
 import org.hibernate.search.annotations.IndexedEmbedded;
 import org.hibernate.validator.NotNull;
 
@@ -53,6 +56,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.hibernate.SearchFieldConstant
  */
 @Entity
 @Table(name = TableNames.PROCEDURES_TABLE)
+@Indexed
 public class ProcedurePE implements IIdHolder, Serializable
 {
     public final static ProcedurePE[] EMPTY_ARRAY = new ProcedurePE[0];
@@ -114,6 +118,7 @@ public class ProcedurePE implements IIdHolder, Serializable
     @SequenceGenerator(name = SequenceNames.PROCEDURE_SEQUENCE, sequenceName = SequenceNames.PROCEDURE_SEQUENCE, allocationSize = 1)
     @Id
     @GeneratedValue(strategy = GenerationType.SEQUENCE, generator = SequenceNames.PROCEDURE_SEQUENCE)
+    @DocumentId
     public final Long getId()
     {
         return id;
@@ -169,8 +174,9 @@ public class ProcedurePE implements IIdHolder, Serializable
         this.inputSamples = inputSamples;
     }
 
-    @OneToMany(fetch = FetchType.LAZY)
-    @JoinColumn(name = ColumnNames.PROCEDURE_PRODUCED_BY_COLUMN, updatable = false)
+    @OneToMany(fetch = FetchType.LAZY, mappedBy = "procedure")
+    @JoinColumn(name = ColumnNames.PROCEDURE_PRODUCED_BY_COLUMN, updatable = true)
+    @ContainedIn
     public Set<DataPE> getData()
     {
         return data;

openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/hibernate/SearchFieldConstants.java

@@ -68,7 +68,7 @@ public final class SearchFieldConstants
      * bridge class or the field name consists of some other non empty elements).
      */
 
-    public static final String PREFIX_PROCEDURE = "";
+    public static final String PREFIX_PROCEDURE = "in" + SEPARATOR;
 
     public static final String PREFIX_EXPERIMENT_ATTACHMENTS = "";
 }

openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/AbstractDAOTest.java

@@ -59,6 +59,8 @@ import ch.systemsx.cisd.openbis.generic.shared.util.UuidUtil;
     { AbstractDAO.class })
 public abstract class AbstractDAOTest extends AbstractTransactionalTestNGSpringContextTests
 {
+    static final String LUCENE_INDEX_PATH = "targets/lucene/indices";
+
     static
     {
         LogInitializer.init();
@@ -66,7 +68,7 @@ public abstract class AbstractDAOTest extends AbstractTransactionalTestNGSpringC
         System.setProperty("database.kind", "test");
         System.setProperty("script-folder", "sourceTest");
         System.setProperty("hibernate.search.index-mode", "NO_INDEX");
-        System.setProperty("hibernate.search.index-base", "sourceTest/lucene/indices");
+        System.setProperty("hibernate.search.index-base", LUCENE_INDEX_PATH);
         System.setProperty("mass-upload-folder", "sourceTest/sql/postgresql");
     }
 

openbis/sourceTest/java/ch/systemsx/cisd/openbis/generic/server/dataaccess/db/HibernateSearchDAOTest.java

@@ -21,30 +21,44 @@ import static org.testng.AssertJUnit.assertFalse;
 import static org.testng.AssertJUnit.assertTrue;
 import static org.testng.AssertJUnit.fail;
 
+import java.io.File;
 import java.util.ArrayList;
 import java.util.Arrays;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Set;
 
+import org.hibernate.classic.Session;
+import org.hibernate.search.FullTextSession;
+import org.hibernate.search.Search;
 import org.testng.AssertJUnit;
+import org.testng.annotations.BeforeClass;
 import org.testng.annotations.DataProvider;
 import org.testng.annotations.Test;
 
 import ch.rinn.restrictions.Friend;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
 import ch.systemsx.cisd.common.test.AssertionUtil;
 import ch.systemsx.cisd.openbis.generic.server.dataaccess.IHibernateSearchDAO;
 import ch.systemsx.cisd.openbis.generic.server.dataaccess.db.search.LuceneQueryBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetSearchCriteria;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetSearchCriterion;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetSearchField;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetSearchFieldKind;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetSearchCriteria;
 import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SearchCriteriaConnection;
 import ch.systemsx.cisd.openbis.generic.shared.dto.DataSetSearchHitDTO;
+import ch.systemsx.cisd.openbis.generic.shared.dto.EntityPropertyPE;
 import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ExperimentPropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.GroupPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.HierarchyType;
+import ch.systemsx.cisd.openbis.generic.shared.dto.IEntityPropertiesHolder;
 import ch.systemsx.cisd.openbis.generic.shared.dto.MaterialPE;
 import ch.systemsx.cisd.openbis.generic.shared.dto.MaterialPropertyPE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.ProjectPE;
 import ch.systemsx.cisd.openbis.generic.shared.dto.PropertyTypePE;
 import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePE;
+import ch.systemsx.cisd.openbis.generic.shared.dto.SamplePropertyPE;
 import ch.systemsx.cisd.openbis.generic.shared.dto.SearchHit;
 
 /**
@@ -57,6 +71,24 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.SearchHit;
 @Friend(toClasses = HibernateSearchDAO.class)
 public final class HibernateSearchDAOTest extends AbstractDAOTest
 {
+    private final static String LUCENE_INDEX_TEMPLATE_PATH = "./sourceTest/lucene/indices";
+
+    @BeforeClass
+    public void setUpIndex()
+    {
+        restoreSearchIndex();
+    }
+
+    // create a fresh copy of the Lucene index
+    private static void restoreSearchIndex()
+    {
+        File targetPath = new File(LUCENE_INDEX_PATH);
+        FileUtilities.deleteRecursively(targetPath);
+        targetPath.mkdirs();
+        File srcPath = new File(LUCENE_INDEX_TEMPLATE_PATH);
+        FileUtilities.copyDirectory(srcPath, targetPath);
+    }
+
     @SuppressWarnings("unused")
     @DataProvider(name = "registratorTerm")
     private final static Object[][] getRegistratorTerm()
@@ -215,7 +247,8 @@ public final class HibernateSearchDAOTest extends AbstractDAOTest
 
     // NOTE: such a check depends strongly on the test database content. Use it only when the better
     // way to check the results is much harder.
-    private void assertCorrectDatasetsFound(DataSetSearchCriteria criteria, DSLoc... expectedLocations)
+    private void assertCorrectDatasetsFound(DataSetSearchCriteria criteria,
+            DSLoc... expectedLocations)
     {
         List<DataSetSearchHitDTO> dataSets = searchForDatasets(criteria);
         AssertJUnit.assertEquals(expectedLocations.length, dataSets.size());
@@ -270,6 +303,26 @@ public final class HibernateSearchDAOTest extends AbstractDAOTest
         assertCorrectDatasetsFound(criteria, DSLoc.LOC1, DSLoc.LOC2, DSLoc.LOC3);
     }
 
+    @Test
+    public final void testSearchForDataSetsSpecificSampleProperty()
+    {
+        String propertyValue = "stuff";
+        DataSetSearchCriterion criterion =
+                mkCriterion(DataSetSearchField.createSampleProperty("USER.COMMENT"), propertyValue);
+        DataSetSearchCriteria criteria = createAndDatasetQuery(criterion);
+        assertCorrectDatasetsFound(criteria, DSLoc.LOC1, DSLoc.LOC2, DSLoc.LOC3);
+    }
+
+    @Test
+    public final void testSearchForDataSetsSimpleField()
+    {
+        DataSetSearchCriterion criterion =
+                mkCriterion(DataSetSearchField.createSimpleField(DataSetSearchFieldKind.PROJECT),
+                        "NEMO");
+        DataSetSearchCriteria criteria = createAndDatasetQuery(criterion);
+        assertCorrectDatasetsFound(criteria, DSLoc.LOC1, DSLoc.LOC3, DSLoc.LOC4, DSLoc.LOC5);
+    }
+
     @Test
     public final void testSearchForDataSetsAnyProperty()
     {
@@ -303,4 +356,185 @@ public final class HibernateSearchDAOTest extends AbstractDAOTest
                 createAndDatasetQuery(criterion1, criterion2, criterion3, criterion4);
         assertCorrectDatasetsFound(criteria, DSLoc.LOC4, DSLoc.LOC5);
     }
+
+    @Test
+    /*
+     * Checks if the dataset search index is properly updated after properties for a connected
+     * sample have changed.
+     */
+    public final void testSearchForDataSetsAfterSamplePropertiesUpdate()
+            throws InterruptedException
+    {
+        String propertyCode = "USER.COMMENT";
+        DataSetSearchCriterion criterion =
+                mkCriterion(DataSetSearchField.createSampleProperty(propertyCode), "stuff");
+
+        DataSetSearchCriteria criteria = createAndDatasetQuery(criterion);
+        assertCorrectDatasetsFound(criteria, DSLoc.LOC1, DSLoc.LOC2, DSLoc.LOC3);
+
+        SamplePE sample = findSample("CP-TEST-3", "CISD");
+
+        String newValue = "Bonanza";
+        changeSampleProperty(sample, propertyCode, newValue);
+
+        flushSearchIndices();
+        assertCorrectDatasetsFound(criteria, DSLoc.LOC2, DSLoc.LOC3);
+        restoreSearchIndex();
+    }
+
+    @Test
+    /*
+     * Checks if the dataset search index is properly updated after properties for a connected
+     * experiment have changed.
+     */
+    public final void testSearchForDataSetsAfterExperimentPropertiesUpdate()
+            throws InterruptedException
+    {
+        String propertyCode = "USER.GENDER";
+        DataSetSearchCriterion criterion =
+                mkCriterion(DataSetSearchField.createExperimentProperty(propertyCode), "female");
+
+        DataSetSearchCriteria criteria = createAndDatasetQuery(criterion);
+        assertCorrectDatasetsFound(criteria, DSLoc.LOC1);
+
+        // This experiment has two datasets. Each of them has "male" value as a gender property.
+        // We change it to "female" and check, if 2 new search results appear.
+        ExperimentPE exp = findExperiment("EXP-TEST-2", "NEMO", "CISD");
+        String newValue = "male";
+        changeExperimentProperty(exp, propertyCode, newValue);
+        flushSearchIndices();
+        assertCorrectDatasetsFound(criteria);
+        restoreSearchIndex();
+    }
+
+    private void flushSearchIndices()
+    {
+        Session currentSession = sessionFactory.getCurrentSession();
+        FullTextSession fullTextSession = Search.getFullTextSession(currentSession);
+        fullTextSession.flushToIndexes();
+    }
+
+    private void flushSession()
+    {
+        sessionFactory.getCurrentSession().flush();
+    }
+
+    private void changeExperimentProperty(ExperimentPE exp, String propertyCode, String newValue)
+    {
+        ExperimentPropertyPE property = findProperty(exp, propertyCode);
+
+        removeProperty(exp, property);
+        flushSession();
+
+        ExperimentPropertyPE newProperty = new ExperimentPropertyPE();
+        copyPropertyWithNewValue(newValue, property, newProperty);
+        addProperty(exp, newProperty);
+        flushSession();
+    }
+
+    private void changeSampleProperty(SamplePE sample, String propertyCode, String newValue)
+    {
+        SamplePropertyPE property = findProperty(sample, propertyCode);
+
+        removeProperty(sample, property);
+        flushSession();
+
+        SamplePropertyPE newProperty = new SamplePropertyPE();
+        copyPropertyWithNewValue(newValue, property, newProperty);
+        addProperty(sample, newProperty);
+        flushSession();
+    }
+
+    private void copyPropertyWithNewValue(String newValue, EntityPropertyPE oldProperty,
+            EntityPropertyPE newProperty)
+    {
+        newProperty.setEntityTypePropertyType(oldProperty.getEntityTypePropertyType());
+        newProperty.setRegistrator(oldProperty.getRegistrator());
+        newProperty.setValue(newValue);
+    }
+
+    private static <T extends EntityPropertyPE> void addProperty(
+            IEntityPropertiesHolder<T> propertiesHolder, T newProperty)
+    {
+        Set<T> properties = getCopiedProperties(propertiesHolder);
+        properties.add(newProperty);
+        propertiesHolder.setProperties(properties);
+    }
+
+    private static <T extends EntityPropertyPE> Set<T> removeProperty(
+            IEntityPropertiesHolder<T> propertiesHolder, T property)
+    {
+        Set<T> properties = getCopiedProperties(propertiesHolder);
+        boolean removed = properties.remove(property);
+        assert removed : "property could not be removed";
+        propertiesHolder.setProperties(properties);
+        return properties;
+    }
+
+    private static <T extends EntityPropertyPE> Set<T> getCopiedProperties(
+            IEntityPropertiesHolder<T> propertiesHolder)
+    {
+        return new HashSet<T>(propertiesHolder.getProperties());
+    }
+
+    private static <T extends EntityPropertyPE> T findProperty(
+            IEntityPropertiesHolder<T> propertiesHolder, String propertyCode)
+    {
+        for (T prop : propertiesHolder.getProperties())
+        {
+            if (prop.getEntityTypePropertyType().getPropertyType().getCode().equals(propertyCode))
+            {
+                return prop;
+            }
+        }
+        fail("property not found: " + propertyCode);
+        return null; // never happens
+    }
+
+    private ExperimentPE findExperiment(String code, String projectCode, String groupCode)
+    {
+        ProjectPE project = findProject(projectCode, groupCode);
+        return findExperiment(code, project);
+    }
+
+    private ExperimentPE findExperiment(String code, ProjectPE project)
+    {
+        ExperimentPE exp = daoFactory.getExperimentDAO().tryFindByCodeAndProject(project, code);
+        assert exp != null : "cannot find experiment: " + code;
+        return exp;
+    }
+
+    private SamplePE findSample(String sampleCode, String groupCode)
+    {
+        GroupPE group = findGroup(groupCode);
+        SamplePE sample = findSample(sampleCode, group);
+        return sample;
+    }
+
+    private SamplePE findSample(String sampleCode, GroupPE group)
+    {
+        SamplePE sample =
+                daoFactory.getSampleDAO().tryFindByCodeAndGroup(sampleCode, group,
+                        HierarchyType.CHILD);
+        assert sample != null : "cannot find sample: " + sampleCode;
+        return sample;
+    }
+
+    private ProjectPE findProject(String code, String groupCode)
+    {
+        ProjectPE result =
+                daoFactory.getProjectDAO().tryFindProject(
+                        daoFactory.getHomeDatabaseInstance().getCode(), groupCode, code);
+        assert result != null : "cannot find the project: " + code;
+        return result;
+    }
+
+    private GroupPE findGroup(String groupCode)
+    {
+        GroupPE group =
+                daoFactory.getGroupDAO().tryFindGroupByCodeAndDatabaseInstance(groupCode,
+                        daoFactory.getHomeDatabaseInstance());
+        assert group != null : "cannot find the group: " + groupCode;
+        return group;
+    }
 }