TimeSeriesDataSetHandlerTest.java

/*
 * Copyright 2009 ETH Zuerich, CISD
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

package eu.basysbio.cisd.dss;

import static eu.basysbio.cisd.dss.DataSetTypeTranslator.DATA_SET_TYPES_KEY;
import static eu.basysbio.cisd.dss.TimeSeriesDataSetHandler.DATA_FILE_TYPE;
import static eu.basysbio.cisd.dss.TimeSeriesDataSetHandler.DATA_SET_PROPERTIES_FILE_NAME_KEY;
import static eu.basysbio.cisd.dss.TimeSeriesDataSetHandler.TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY;
import static eu.basysbio.cisd.dss.TimeSeriesDataSetHandler.TRANSLATION_KEY;

import java.io.File;
import java.io.IOException;
import java.io.PrintWriter;
import java.util.Arrays;
import java.util.Collections;
import java.util.LinkedHashSet;
import java.util.List;
import java.util.Properties;
import java.util.Set;

import org.apache.commons.io.IOUtils;
import org.hamcrest.BaseMatcher;
import org.hamcrest.Description;
import org.jmock.Expectations;
import org.jmock.Mockery;
import org.testng.annotations.AfterMethod;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;

import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
import ch.systemsx.cisd.common.Constants;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.test.AssertionUtil;
import ch.systemsx.cisd.etlserver.utils.Column;
import ch.systemsx.cisd.etlserver.utils.TableBuilder;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.TechId;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetTypeWithVocabularyTerms;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityProperty;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Group;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ListSampleCriteria;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.NewSample;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Project;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PropertyType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PropertyTypeWithVocabulary;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.VocabularyTerm;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ExperimentIdentifier;

/**
 * 
 *
 * @author Franz-Josef Elmer
 */
public class TimeSeriesDataSetHandlerTest extends AbstractFileSystemTestCase
{

    private static final String DATA_SET_PROPERTIES_FILE = "data-set-properties.txt";
    private static final String SAMPLE_EX_200 = "GM_BR_B1_EX_200";
    private static final String DATA_SET_EX_200 = SAMPLE_EX_200 + ".T1.CE.MetaboliteLCMS.Value[mM].Log10.NB.NC";
    private static final String SAMPLE_EX_7200 = "GM_BR_B1_EX_7200";
    private static final String DATA_SET_EX_7200 = SAMPLE_EX_7200 + ".T2.CE.b.Value[mM].LIN.NB.NC";
    private static final String GROUP_CODE = "G1";
    private static final long EXP_ID = 42L;
    private static final String PROJECT_CODE = "p1";
    private Mockery context;
    private IEncapsulatedOpenBISService service;
    private File dropBox;


    @BeforeMethod
    public void beforeMethod() throws Exception
    {
        super.setUp();
        LogInitializer.init();
        context = new Mockery();
        service = context.mock(IEncapsulatedOpenBISService.class);
        dropBox = new File(workingDirectory, "drop-box");
        dropBox.mkdirs();
    }
    
    @AfterMethod
    public void tearDown()
    {
        // To following line of code should also be called at the end of each test method.
        // Otherwise one does not known which test failed.
        context.assertIsSatisfied();
    }

    @Test
    public void testMissingDropBoxProperty()
    {
        try
        {
            new TimeSeriesDataSetHandler(new Properties(), service);
            fail("ConfigurationFailureException expected");
        } catch (ConfigurationFailureException ex)
        {
            assertEquals("Given key '" + TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY
                    + "' not found in properties '[]'", ex.getMessage());
        }
        
        context.assertIsSatisfied();
    }
    
    @Test
    public void testMissingDataSetProertiesFileNameProperty()
    {
        try
        {
            Properties properties = new Properties();
            properties.setProperty(TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY, dropBox.getAbsolutePath());
            new TimeSeriesDataSetHandler(properties, service);
            fail("ConfigurationFailureException expected");
        } catch (ConfigurationFailureException ex)
        {
            assertEquals(
                    "Given key '" + DATA_SET_PROPERTIES_FILE_NAME_KEY
                            + "' not found in properties '["
                            + TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY + "]'", ex.getMessage());
        }
        
        context.assertIsSatisfied();
    }
    
    @Test
    public void testMissingDataSetTypesProperty()
    {
        try
        {
            Properties properties = new Properties();
            properties.setProperty(TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY, dropBox.getAbsolutePath());
            properties.setProperty(DATA_SET_PROPERTIES_FILE_NAME_KEY, "p.txt");
            new TimeSeriesDataSetHandler(properties, service);
            fail("ConfigurationFailureException expected");
        } catch (ConfigurationFailureException ex)
        {
            AssertionUtil.assertContains("Given key '" + DATA_SET_TYPES_KEY + "'", ex.getMessage());
        }
        
        context.assertIsSatisfied();
    }
    
    @Test
    public void testWrongDataSetType() throws IOException
    {
        Properties properties = new Properties();
        properties.setProperty(TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY, dropBox.getAbsolutePath());
        properties.setProperty(DATA_SET_PROPERTIES_FILE_NAME_KEY, DATA_SET_PROPERTIES_FILE);
        properties.setProperty(TRANSLATION_KEY + DATA_SET_TYPES_KEY, "a, b");
        properties.setProperty(TRANSLATION_KEY + "a", "Alpha");
        prepareDataSetPropertiesValidator("Alpha", "B");
        TimeSeriesDataSetHandler handler = new TimeSeriesDataSetHandler(properties, service);
        File file = createDataExample();
        DataSetInformation dataSetInformation = createDataSetInformation("BLABLA");
        dataSetInformation.setExperimentIdentifier(new ExperimentIdentifier(PROJECT_CODE, "exp1"));

        try
        {
            handler.handle(file, dataSetInformation);
            fail("UserFailureException expected");
        } catch (UserFailureException ex)
        {
            assertEquals("Data has to be uploaded for data set type TIME_SERIES instead of "
                    + "BLABLA.", ex.getMessage());
        }
        
        context.assertIsSatisfied();
    }
    
    @Test
    public void testMissingExperiment() throws IOException
    {
        Properties properties = new Properties();
        properties.setProperty(TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY, dropBox.getAbsolutePath());
        properties.setProperty(DATA_SET_PROPERTIES_FILE_NAME_KEY, DATA_SET_PROPERTIES_FILE);
        properties.setProperty(TRANSLATION_KEY + DATA_SET_TYPES_KEY, "a, b");
        properties.setProperty(TRANSLATION_KEY + "a", "Alpha");
        prepareDataSetPropertiesValidator("Alpha", "B");
        TimeSeriesDataSetHandler handler = new TimeSeriesDataSetHandler(properties, service);
        
        File file = createDataExample();
        
        try
        {
            handler.handle(file, createDataSetInformation(TimeSeriesDataSetHandler.DATA_SET_TYPE));
            fail("UserFailureException expected");
        } catch (UserFailureException ex)
        {
            assertEquals("Data set should be registered for an experiment and not for a sample.", ex.getMessage());
        }
        
        context.assertIsSatisfied();
    }
    
    @Test
    public void testWrongExperiment() throws IOException
    {
        Properties properties = new Properties();
        properties.setProperty(TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY, dropBox.getAbsolutePath());
        properties.setProperty(DATA_SET_PROPERTIES_FILE_NAME_KEY, DATA_SET_PROPERTIES_FILE);
        properties.setProperty(TRANSLATION_KEY + DATA_SET_TYPES_KEY, "MetaboliteLCMS");
        properties.setProperty(TRANSLATION_KEY + "MetaboliteLCMS", "METABOLITE_LCMS");
        prepareDataSetPropertiesValidator("METABOLITE_LCMS");
        TimeSeriesDataSetHandler handler = new TimeSeriesDataSetHandler(properties, service);
        File file = createDataExample();
        DataSetInformation dataSetInformation =
                createDataSetInformation(TimeSeriesDataSetHandler.DATA_SET_TYPE);
        dataSetInformation.setExperimentIdentifier(new ExperimentIdentifier(PROJECT_CODE, "exp1"));

        try
        {
            handler.handle(file, dataSetInformation);
            fail("UserFailureException expected");
        } catch (UserFailureException ex)
        {
            assertEquals("Data should be uploaded for experiment 'GM_BR_B1' instead of 'EXP1'.", ex
                    .getMessage());
        }

        context.assertIsSatisfied();
    }

    @Test
    public void testDataSetAlreadyExists()
    {
        Properties properties = new Properties();
        properties.setProperty(TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY, dropBox.getAbsolutePath());
        properties.setProperty(DATA_SET_PROPERTIES_FILE_NAME_KEY, DATA_SET_PROPERTIES_FILE);
        properties.setProperty(TRANSLATION_KEY + DATA_SET_TYPES_KEY, "b");
        prepareDataSetPropertiesValidator("B");
        TimeSeriesDataSetHandler handler = new TimeSeriesDataSetHandler(properties, service);
        File file = createDataExample();
        context.checking(new Expectations()
            {
                {
                    one(service).tryToGetExperiment(new ExperimentIdentifier(PROJECT_CODE, "GM_BR_B1"));
                    will(returnValue(createExperiment("GM_BR_B1")));
                    
                    one(service).listSamples(with(any(ListSampleCriteria.class)));
                    Sample sample200 = createSample(SAMPLE_EX_200);
                    will(returnValue(Arrays.<Sample> asList(sample200)));
                    
                    one(service).listDataSetsBySampleID(sample200.getId(), true);
                    will(returnValue(Arrays.asList(createData("ds0", "T0"), createData("ds1", "T1"))));
                }
                
            });
        
        DataSetInformation dataSetInformation = createDataSetInformation(TimeSeriesDataSetHandler.DATA_SET_TYPE);
        dataSetInformation.setExperimentIdentifier(new ExperimentIdentifier(PROJECT_CODE, "GM_BR_B1"));
        try
        {
            handler.handle(file, dataSetInformation);
            fail("UserFailureException expected");
        } catch (UserFailureException ex)
        {
            assertEquals(
                    "For data column 'GM::BR::B1::200::EX::T1::CE::MetaboliteLCMS::Value[mM]::Log10::NB::NC' "
                            + "the data set 'ds1' has already been registered.", ex.getMessage());
        }
        
        context.assertIsSatisfied();
    }
    
    
    @Test
    public void test()
    {
        Properties properties = new Properties();
        properties.setProperty(TIME_POINT_DATA_SET_DROP_BOX_PATH_KEY, dropBox.getAbsolutePath());
        properties.setProperty(DATA_SET_PROPERTIES_FILE_NAME_KEY, DATA_SET_PROPERTIES_FILE);
        properties.setProperty(TRANSLATION_KEY + DATA_SET_TYPES_KEY, "MetaboliteLCMS, b");
        properties.setProperty(TRANSLATION_KEY + "MetaboliteLCMS", "METABOLITE_LCMS");
        prepareDataSetPropertiesValidator("METABOLITE_LCMS", "B");
        TimeSeriesDataSetHandler handler = new TimeSeriesDataSetHandler(properties, service);
        File file = createDataExample();
        context.checking(new Expectations()
        {
            {
                    exactly(2).of(service).tryToGetExperiment(
                            new ExperimentIdentifier(PROJECT_CODE, "GM_BR_B1"));
                    will(returnValue(createExperiment("GM_BR_B1")));

                    exactly(2).of(service).listSamples(with(new BaseMatcher<ListSampleCriteria>()
                        {
                            private TechId experimentId;

                            public void describeTo(Description description)
                            {
                                description.appendText("Experiment ID: expected: " + EXP_ID
                                        + ", actual: " + experimentId);
                            }

                            public boolean matches(Object item)
                            {
                                if (item instanceof ListSampleCriteria)
                                {
                                    experimentId = ((ListSampleCriteria) item).getExperimentId();
                                    return experimentId.getId().equals(EXP_ID);
                                }
                                return false;
                            }
                        }));
                    Sample sample200 = createSample(SAMPLE_EX_200);
                    will(returnValue(Arrays.<Sample> asList(sample200)));

                    one(service).listDataSetsBySampleID(sample200.getId(), true);
                    will(returnValue(Arrays.asList()));

                    final NewSample sample = createNewSample(SAMPLE_EX_7200);
                    one(service).registerSample(with(new BaseMatcher<NewSample>()
                        {
                            public boolean matches(Object item)
                            {
                                if (item instanceof NewSample)
                                {
                                    NewSample actualSample = (NewSample) item;
                                    assertEquals(sample.getIdentifier(), actualSample.getIdentifier());
                                    IEntityProperty[] p = actualSample.getProperties();
                                    assertEquals(1, p.length);
                                    assertEquals("7200", p[0].getValue());
                                    assertEquals("TIME_POINT", p[0].getPropertyType().getCode());
                                    return true;
                                }
                                return false;
                            }

                    public void describeTo(Description description)
                    {
                        description.appendValue(sample);
                    }
                        }));
                will(returnValue(1234l));
                
                one(service).listDataSetsBySampleID(1234l, true);
                will(returnValue(Arrays.asList()));
            }
        });
        
        DataSetInformation dataSetInformation = createDataSetInformation(TimeSeriesDataSetHandler.DATA_SET_TYPE);
        dataSetInformation.setExperimentIdentifier(new ExperimentIdentifier(PROJECT_CODE, "GM_BR_B1"));
        handler.handle(file, dataSetInformation);
        
        File dataSet = new File(dropBox, DATA_SET_EX_200);
        assertEquals(true, dataSet.isDirectory());
        File dataFile = new File(dataSet, "METABOLITE_LCMS" + DATA_FILE_TYPE);
        List<String> data = FileUtilities.loadToStringList(dataFile);
        assertEquals("ID\tHumanReadable\tGM::BR::B1::200::EX::T1::CE::MetaboliteLCMS::Value[mM]::Log10::NB::NC", data.get(0));
        assertEquals("CHEBI:15721\tsedoheptulose 7-phosphate\t0.34", data.get(1));
        assertEquals("CHEBI:18211\tcitrulline\t0.87", data.get(2));
        assertEquals(3, data.size());
        File dataSetPropertiesFile = new File(dataSet, DATA_SET_PROPERTIES_FILE);
        List<String> dataSetProperties = FileUtilities.loadToStringList(dataSetPropertiesFile);
        Collections.sort(dataSetProperties);
        assertEquals(
                "[BI_ID\tNB, CEL_LOC\tCE, CG\tNC, SCALE\tLOG10, TECHNICAL_REPLICATE_CODE\tT1, "
                + "TIME_SERIES_DATA_SET_TYPE\tMetaboliteLCMS, VALUE_TYPE\tValue[mM], "
                + "property\tvalue]", dataSetProperties.toString());
        File markerFile = new File(dropBox, Constants.IS_FINISHED_PREFIX + DATA_SET_EX_200);
        assertEquals(true, markerFile.exists());
        
        dataSet = new File(dropBox, DATA_SET_EX_7200);
        assertEquals(true, dataSet.isDirectory());
        dataFile = new File(dataSet, "B" + DATA_FILE_TYPE);
        data = FileUtilities.loadToStringList(dataFile);
        assertEquals("ID\tHumanReadable\tGM::BR::B1::+7200::EX::T2::CE::b::Value[mM]::LIN::NB::NC", data.get(0));
        assertEquals("CHEBI:15721\tsedoheptulose 7-phosphate\t0.799920281", data.get(1));
        assertEquals("CHEBI:18211\tcitrulline\t1.203723714", data.get(2));
        assertEquals(3, data.size());
        dataSetPropertiesFile = new File(dataSet, DATA_SET_PROPERTIES_FILE);
        dataSetProperties = FileUtilities.loadToStringList(dataSetPropertiesFile);
        Collections.sort(dataSetProperties);
        assertEquals(
                "[BI_ID\tNB, CEL_LOC\tCE, CG\tNC, SCALE\tLIN, TECHNICAL_REPLICATE_CODE\tT2, "
                + "TIME_SERIES_DATA_SET_TYPE\tb, VALUE_TYPE\tValue[mM], "
                + "property\tvalue]", dataSetProperties.toString());
        markerFile = new File(dropBox, Constants.IS_FINISHED_PREFIX + DATA_SET_EX_7200);
        assertEquals(true, markerFile.exists());
        
        context.assertIsSatisfied();
    }
    
    private void prepareDataSetPropertiesValidator(final String... dataSetTypes)
    {
        context.checking(new Expectations()
            {
                {
                    for (String dataSetTypeCode : dataSetTypes)
                    {
                        one(service).getDataSetType(dataSetTypeCode);
                        DataSetTypeWithVocabularyTerms type = new DataSetTypeWithVocabularyTerms();
                        type.setCode(dataSetTypeCode);
                        for (TimePointPropertyType timePointPropertyType : TimePointPropertyType.values())
                        {
                            PropertyTypeWithVocabulary propertyType = new PropertyTypeWithVocabulary();
                            propertyType.setCode(timePointPropertyType.toString());
                            if (timePointPropertyType.isVocabulary())
                            {
                                propertyType.setTerms(createTerms("CE", "LIN", "LOG10"));
                                
                            }
                            type.addPropertyType(propertyType);

                        }
                        will(returnValue(type));
                    }
                }
                
                private Set<VocabularyTerm> createTerms(String...terms)
                {
                    LinkedHashSet<VocabularyTerm> set = new LinkedHashSet<VocabularyTerm>();
                    for (String term : terms)
                    {
                        set.add(createTerm(term));
                    }
                    return set;
                }
                
                private VocabularyTerm createTerm(String code)
                {
                    VocabularyTerm vocabularyTerm = new VocabularyTerm();
                    vocabularyTerm.setCode(code);
                    return vocabularyTerm;
                }
            });
    }

    private File createDataExample()
    {
        TableBuilder builder =
                new TableBuilder("ID", "HumanReadable",
                        "GM::BR::B1::200::EX::T1::CE::MetaboliteLCMS::Value[mM]::Log10::NB::NC",
                        "GM::BR::B1::+7200::EX::T2::CE::b::Value[mM]::LIN::NB::NC");
        builder.addRow("CHEBI:15721", "sedoheptulose 7-phosphate", "0.34", "0.799920281");
        builder.addRow("CHEBI:18211", "citrulline", "0.87", "1.203723714");
        File file = new File(workingDirectory, "data.txt");
        write(builder, file);
        return file;
    }
    
    private void write(TableBuilder builder, File file)
    {
        PrintWriter writer = null;
        try
        {
            writer = new PrintWriter(file);
            List<Column> columns = builder.getColumns();
            String delim = "";
            for (Column column : columns)
            {
                writer.print(delim);
                writer.print(column.getHeader());
                delim = "\t";
            }
            writer.println();
            int numberOfRows = columns.get(0).getValues().size();
            for (int i = 0; i < numberOfRows; i++)
            {
                delim = "";
                for (Column column : columns)
                {
                    writer.print(delim);
                    writer.print(column.getValues().get(i));
                    delim = "\t";
                }
                writer.println();
            }
        } catch (IOException ex)
        {
            throw CheckedExceptionTunnel.wrapIfNecessary(ex);
        } finally
        {
            IOUtils.closeQuietly(writer);
        }
    }

    private Experiment createExperiment(String code)
    {
        Experiment experiment = new Experiment();
        experiment.setId(EXP_ID);
        experiment.setCode(code);
        Project project = new Project();
        project.setCode(PROJECT_CODE);
        Group group = new Group();
        group.setIdentifier(GROUP_CODE);
        project.setGroup(group);
        experiment.setProject(project);
        return experiment;
    }

    private NewSample createNewSample(String sampleCode)
    {
        NewSample sample = new NewSample();
        sample.setContainerIdentifier(sampleCode);
        sample.setIdentifier(GROUP_CODE + "/" + sampleCode);
        return sample;
    }
    
    private Sample createSample(String sampleCode)
    {
        Sample sample = new Sample();
        sample.setCode(sampleCode);
        sample.setId(new Long(sampleCode.length()));
        return sample;
    }
    
    private DataSetInformation createDataSetInformation(String dataSetTypeCode)
    {
        DataSetInformation dataSetInformation = new DataSetInformation();
        DataSetType dataSetType = new DataSetType();
        dataSetType.setCode(dataSetTypeCode);
        dataSetInformation.setDataSetType(dataSetType);
        return dataSetInformation;
    }

    private ExternalData createData(String code, String technicalReplicateCode)
    {
        ExternalData externalData = new ExternalData();
        externalData.setCode(code);
        EntityProperty tr = createProperty(TimePointPropertyType.TECHNICAL_REPLICATE_CODE, technicalReplicateCode);
        EntityProperty biID = createProperty(TimePointPropertyType.BI_ID, "NB");
        EntityProperty cg = createProperty(TimePointPropertyType.CG, "NC");
        EntityProperty vt = createProperty(TimePointPropertyType.VALUE_TYPE, "Value[mM]");
        EntityProperty scale = createProperty(TimePointPropertyType.SCALE, "LOG10");
        EntityProperty celLoc = createProperty(TimePointPropertyType.CEL_LOC, "CE");
        EntityProperty t = createProperty(TimePointPropertyType.TIME_SERIES_DATA_SET_TYPE, "MetaboliteLCMS");
        externalData.setDataSetProperties(Arrays.<IEntityProperty>asList(tr, biID, cg, vt, scale, celLoc, t));
        return externalData;
    }
    
    private EntityProperty createProperty(TimePointPropertyType type, String value)
    {
        return createProperty(type.toString(), value);
    }
    
    private EntityProperty createProperty(String code, String value)
    {
        EntityProperty entityProperty = new EntityProperty();
        PropertyType propertyType = new PropertyType();
        propertyType.setCode(code);
        entityProperty.setPropertyType(propertyType);
        entityProperty.setValue(value);
        return entityProperty;
    }
}