OpenBISScreeningML.java

/*
 * Copyright 2010 ETH Zuerich, CISD
 *
 * Licensed under the Apache License, Version 2.0 (the "License");
 * you may not use this file except in compliance with the License.
 * You may obtain a copy of the License at
 *
 *      http://www.apache.org/licenses/LICENSE-2.0
 *
 * Unless required by applicable law or agreed to in writing, software
 * distributed under the License is distributed on an "AS IS" BASIS,
 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
 * See the License for the specific language governing permissions and
 * limitations under the License.
 */

import java.io.BufferedOutputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Set;
import java.util.TreeSet;

import ch.systemsx.cisd.openbis.dss.client.api.v1.IDataSetDss;
import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.FileInfoDssDTO;
import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetMetadataDTO;
import ch.systemsx.cisd.openbis.generic.client.cli.Login;
import ch.systemsx.cisd.openbis.plugin.screening.client.api.v1.IScreeningOpenbisServiceFacade;
import ch.systemsx.cisd.openbis.plugin.screening.client.api.v1.IScreeningOpenbisServiceFacadeFactory;
import ch.systemsx.cisd.openbis.plugin.screening.client.api.v1.ScreeningOpenbisServiceFacade.IImageOutputStreamProvider;
import ch.systemsx.cisd.openbis.plugin.screening.client.api.v1.ScreeningOpenbisServiceFacadeFactory;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.ExperimentIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVector;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVectorDataset;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVectorDatasetReference;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVectorDatasetWellReference;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.FeatureVectorWithDescription;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Geometry;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.IFeatureCodesProvider;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.ImageDatasetMetadata;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.ImageDatasetReference;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.MaterialIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.MaterialTypeIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.Plate;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.PlateIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.PlateImageReference;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.PlateWellMaterialMapping;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.WellIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.api.v1.dto.WellPosition;

/**
 * Simple Matlab interface for openBIS for Screening. It is meant to be used in one Matlab session
 * at a time, i.e. it is <i>not</i> multi-threading safe.
 * <p>
 * While written in Java, the API is idiomatic for Matlab, i.e. values are returned as
 * multi-dimensional arrays. For the <code>get...</code> and <code>load...</code> methods the first
 * index will contain the actual data, while the second index will contain per-row annotations. For
 * <code>getFeatureMatrix</code>, the third index contains per-column annotations. This allows
 * simple access with Matlab's slicing operator, see doc of e.g. {@link #getFeatureMatrix(String)}.
 * <p>
 * A typical Matlab session looks like:
 * 
 * <pre>
 * % Add the API jar file to the classpath
 * javaaddpath('/home/brinn/matlab/openbis_screening_api-batteries_included.jar')
 * % Login to server
 * OpenBISScreeningML.login('user', 'secret', 'https://www.infectome.org')
 * 
 * % ...perform calls on the server...
 * 
 * % Logout to close the session on the server
 * OpenBISScreeningML.logout()
 * </pre>
 * 
 * <i>Note: using this login your password will end up in the Matlab command history. An alternative
 * that avoids this is to call the {@link ch.systemsx.cisd.openbis.generic.client.cli.Login} class.
 * Logging in on the console will grant this class access to the openBIS server.</i>
 * <p>
 * To learn the API one needs to understand three basic notions: code, augmented code and perm id.
 * Space, project, experiment, plate and well have their own <b>code</b>, which is unique only in
 * the context of the parent.<br>
 * That's why one needs <b>augmented code</b> to point e.g. to one experiment, because two different
 * projects can have experiments with the same code.<br>
 * Such an augmented code for experiment has a form of "/space-code/project-code/experiment-code".<br>
 * For plate it has a form of "/space-code/plate-code" (note that plate code is unique on the space
 * level). <br>
 * The drawback of an augmented code is that it's not persistent. If someone e.g. moves the
 * experiment from one space to the other augmented code of the experiment becomes invalid. That is
 * why experiments, plates and datasets have <b>perm id</b> (permament identifier) which never
 * change and allow to refer to them with one "magic" identifier, e.g. 20110516124520378-737166.
 * </p>
 * 
 * @author Bernd Rinn
 */
public class OpenBISScreeningML
{
    private static interface ITileNumberIterable extends Iterable<Integer>
    {
        public void setMaximumNumberOfTiles(int numberOfTiles);

        public int getMaximumNumberOfTiles();
    }

    private static final class ImageReferenceAndFile
    {
        private final PlateImageReference imageReference;

        private final File imageFile;

        private BufferedOutputStream outputStream;

        ImageReferenceAndFile(PlateImageReference imageReference, File imageFile)
        {
            this.imageReference = imageReference;
            this.imageFile = imageFile;
        }

        public PlateImageReference getImageReference()
        {
            return imageReference;
        }

        public File getImageFile()
        {
            return imageFile;
        }

        public OutputStream open() throws IOException
        {
            if (outputStream == null)
            {
                outputStream = new BufferedOutputStream(new FileOutputStream(imageFile));
            }
            return outputStream;
        }

        public void close() throws IOException
        {
            if (outputStream != null)
            {
                outputStream.close();
            }
            outputStream = null;
        }
    }

    private static final class IndexSet<T extends Comparable<T>>
    {
        private final Map<T, Integer> indexMap = new HashMap<T, Integer>();
        private final Set<T> set = new TreeSet<T>();
        
        public void add(T item)
        {
            set.add(item);
        }
        
        public int getIndex(T item)
        {
            if (set.size() != indexMap.size())
            {
                List<T> items = new ArrayList<T>(set);
                for (int i = 0; i < items.size(); i++)
                {
                    indexMap.put(items.get(i), i);
                }
            }
            Integer index = indexMap.get(item);
            if (index == null)
            {
                throw new IllegalArgumentException("Unknown item: " + item);
            }
            return index;
        }
        
        public int size()
        {
            return set.size();
        }
    }

    static final String DATASETS_FOLDER = "openbis_datasets";

    private static File temporarySessionDir;

    private static Map<PlateImageReference, File> loadedImages;

    static IScreeningOpenbisServiceFacadeFactory facadeFactory =
            ScreeningOpenbisServiceFacadeFactory.INSTANCE;

    private static IScreeningOpenbisServiceFacade openbis = null;

    private static List<ExperimentIdentifier> experiments = null;

    private static List<Plate> plates = null;

    private static Map<String, List<Plate>> experimentToPlateMap =
            new HashMap<String, List<Plate>>();

    private static Map<String, ExperimentIdentifier> experimentCodeToExperimentMap =
            new HashMap<String, ExperimentIdentifier>();

    private static Map<String, Plate> plateCodeToPlateMap = new HashMap<String, Plate>();

    private OpenBISScreeningML()
    {
        // Not to be constructed.
    }

    //
    // Versioning
    //

    /**
     * The version of the API.
     */
    public static final String VERSION = "1";

    /**
     * The required version ("major.minor") of the screening API on the openBIS application server.
     */
    public static final String REQUIRES_OPENBIS_AS_API = "1.7";

    /**
     * The required version ("major.minor") of the screening API on the openBIS datastore server.
     */
    public static final String REQUIRES_OPENBIS_DSS_API = "1.1";

    private static File dataSetsDir;

    /**
     * Root temporary directory for data sets and images. By default <code>java.io.tmpdir</code> is
     * used.
     */
    static File tempDir = new File(System.getProperty("java.io.tmpdir"));

    static final String TEMP_DIR_PREFIX = "openbis_";

    static final String TEMP_DIR_POSTFIX = "_temp_dir";

    //
    // Authentication methods
    //

    /**
     * Login to the openBIS server given as <var>url</var>.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * OpenBISScreeningML.login('user', 'secret', 'https://www.infectome.org')
     * </pre>
     * 
     * @param user The user id on the server
     * @param password The password on the server
     * @param url The URL, e.g. <code>https://www.infectome.org</var>
     */
    public static void login(String user, String password, String url)
    {
        IScreeningOpenbisServiceFacade facade = facadeFactory.tryToCreate(user, password, url);
        if (facade == null)
        {
            throw new RuntimeException("Login failed.");
        }
        init(facade);
    }

    private static void init(IScreeningOpenbisServiceFacade openBisFacade)
    {
        openbis = openBisFacade;
        dataSetsDir = new File(tempDir, DATASETS_FOLDER);
        if (dataSetsDir.isDirectory() == false && dataSetsDir.mkdirs() == false)
        {
            throw new RuntimeException("Couldn't create a data set directory.");
        }
        temporarySessionDir =
                new File(tempDir, TEMP_DIR_PREFIX + System.currentTimeMillis() / 1000
                        + TEMP_DIR_POSTFIX);
        if (temporarySessionDir.mkdirs() == false)
        {
            throw new RuntimeException("Couldn't create a temporary directory.");
        }
        temporarySessionDir.deleteOnExit();
        loadedImages = new HashMap<PlateImageReference, File>();
        experiments = openbis.listExperiments();
        experimentCodeToExperimentMap.clear();
        for (ExperimentIdentifier e : experiments)
        {
            experimentCodeToExperimentMap.put(e.getAugmentedCode(), e);
        }
        plates = openbis.listPlates();
        plateCodeToPlateMap.clear();
        experimentToPlateMap.clear();
        for (Plate p : plates)
        {
            final String plateCode = p.getAugmentedCode();
            plateCodeToPlateMap.put(plateCode, p);
            final String experimentCode = p.getExperimentIdentifier().getAugmentedCode();
            List<Plate> experimentPlates = experimentToPlateMap.get(experimentCode);
            if (experimentPlates == null)
            {
                experimentPlates = new ArrayList<Plate>();
                experimentToPlateMap.put(experimentCode, experimentPlates);
            }
            experimentPlates.add(p);
        }
    }

    /**
     * Logs out and closes the session on the server.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * OpenBISScreeningML.logout()
     * </pre>
     */
    public static void logout()
    {
        if (openbis == null)
        {
            return;
        }
        openbis.logout();
        if (Login.OPENBIS_TOKEN_FILE.exists())
        {
            Login.OPENBIS_TOKEN_FILE.delete();
        }
        delete(temporarySessionDir);
        openbis = null;
    }

    private static void delete(File file)
    {
        if (file.isDirectory())
        {
            File[] files = file.listFiles();
            for (File child : files)
            {
                delete(child);
            }
        }
        file.delete();
    }

    //
    // Information methods
    //

    /**
     * Lists all experiment.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Get the experiments
     * exps = OpenBISScreeningML.listExperiments();
     * % How many experiments do we have?
     * length(exps)
     * % Get all information about experiment 3
     * exp3 = exps(3,:)
     * % Get the perm ids for all experiments
     * permids = exps(:,2)
     * </pre>
     * 
     * @return Each row contains information about one experiment:
     *         <p>
     *         <code>{ experiment augmented code, experiment perm id, experiment space code, 
     *         experiment project code, experiment code }</code>
     */
    public static Object[][] listExperiments()
    {
        checkLoggedIn();
        final Object[][] result = new Object[experiments.size()][5];
        for (int i = 0; i < experiments.size(); ++i)
        {
            final Object[] annotations =
                    new Object[]
                        { experiments.get(i).getAugmentedCode(), experiments.get(i).getPermId(),
                                experiments.get(i).getSpaceCode(),
                                experiments.get(i).getProjectCode(),
                                experiments.get(i).getExperimentCode() };
            System.arraycopy(annotations, 0, result[i], 0, annotations.length);
        }
        return result;
    }

    /**
     * Lists all plates.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Get the plates
     * plates = OpenBISScreeningML.listPlates();
     * % How many plates do we have?
     * length(plates)
     * % Get all information about plate 2
     * plate2 = plates(2,:)
     * % Get the simple plate codes for all plates
     * codes = plates(:,4)
     * </pre>
     * 
     * @return Each row contains information about one plate:
     *         <p>
     *         <code>{ plate augmented code, plate perm id, plate space code, plate code, 
     *         experiment augmented code, experiment perm id, experiment space code, 
     *         experiment project code, experiment code }</code>
     */
    public static Object[][] listPlates()
    {
        checkLoggedIn();
        return listPlates(plates);
    }

    /**
     * Lists the plates of <var>experiment</var>.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Get the plates of experiment MYEXP in project PROJ of space SPACE
     * plates = OpenBISScreeningML.listPlates('/SPACE/PROJ/MYEXP');
     * % How many plates do we have?
     * length(plates)
     * % Get all information about plate 2
     * plate2 = plates(2,:)
     * % Get the augmented plate codes for all plates
     * acodes = plates(:,1)
     * </pre>
     * 
     * @param experiment The augmented code of the experiment to list the plates for
     * @return Each row contains information about one plate:
     *         <p>
     *         <code>{ plate augmented code, plate perm id, plate space code, plate code, 
     *         experiment augmented code, experiment perm id, experiment space code, 
     *         experiment project code, experiment code }</code>
     */
    public static Object[][] listPlates(String experiment)
    {
        checkLoggedIn();
        final List<Plate> experimentPlates = experimentToPlateMap.get(experiment);
        if (experimentPlates == null)
        {
            throw new RuntimeException("No experiment with that code found.");
        }
        return listPlates(experimentPlates);
    }

    private static Object[][] listPlates(final List<Plate> list)
    {
        final Object[][] result = new Object[list.size()][9];
        for (int i = 0; i < list.size(); ++i)
        {
            final Object[] annotations =
                    new Object[]
                        { list.get(i).getAugmentedCode(), plates.get(i).getPermId(),
                                list.get(i).tryGetSpaceCode(), plates.get(i).getPlateCode(),
                                list.get(i).getExperimentIdentifier().getAugmentedCode(),
                                list.get(i).getExperimentIdentifier().getPermId(),
                                list.get(i).getExperimentIdentifier().getSpaceCode(),
                                list.get(i).getExperimentIdentifier().getProjectCode(),
                                list.get(i).getExperimentIdentifier().getExperimentCode(), };
            System.arraycopy(annotations, 0, result[i], 0, annotations.length);
        }
        return result;
    }

    /**
     * Returns the properties of specified well for specified plate.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Get properties for well A03 of plate P005 in space SPACE
     * properties = OpenBISScreeningML.getWellProperties('/SPACE/P005', 1, 3)
     * % Get property type code of first property
     * properties(1,1)
     * % Get property value of first property
     * properties(1,2)
     * </pre>
     * 
     * @param augmentedPlateCode The augmented plate code
     * @param row The row in the plate to get the well properties for
     * @param column The column in the plate to get the well properties for
     * @return A two dimensional array where the first column contains the property codes and the
     *         second column the corresponding property values.
     */
    public static Object[][] getWellProperties(String augmentedPlateCode, int row, int column)
    {
        checkLoggedIn();
        WellPosition wellPosition = new WellPosition(row, column);
        WellIdentifier wellIdentifier = getWell(augmentedPlateCode, wellPosition);
        List<Map.Entry<String, String>> list =
                new ArrayList<Map.Entry<String, String>>(openbis.getWellProperties(wellIdentifier)
                        .entrySet());
        Object[][] result = new Object[list.size()][2];
        for (int i = 0; i < list.size(); i++)
        {
            result[i] = new Object[]
                { list.get(i).getKey(), list.get(i).getValue() };
        }
        return result;
    }

    /**
     * Updates properties of specified well for specified plate.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Updates properties DESCRIPTION and NUMBER for well A03 of plate P005 in space SPACE
     * properties = {'DESCRIPTION' 'hello example'; 'NUMBER' 3.14}
     * OpenBISScreeningML.updateWellProperties('/SPACE/P005', 1, 3, properties)
     * </pre>
     * 
     * @param augmentedPlateCode The augmented plate code
     * @param row The row in the plate to get the well properties for
     * @param column The column in the plate to get the well properties for
     * @param properties A two dimensional array where the first column contains the property codes
     *            and the second column the corresponding property values.
     */
    public static void updateWellProperties(String augmentedPlateCode, int row, int column,
            Object[][] properties)
    {
        checkLoggedIn();
        WellPosition wellPosition = new WellPosition(row, column);
        WellIdentifier wellIdentifier = getWell(augmentedPlateCode, wellPosition);
        openbis.updateWellProperties(wellIdentifier, createMap(properties));
    }

    private static WellIdentifier getWell(String augmentedPlateCode, WellPosition wellPosition)
    {
        Plate plate = getPlate(augmentedPlateCode);
        List<WellIdentifier> wells = openbis.listPlateWells(plate);
        for (WellIdentifier wellIdentifier : wells)
        {
            if (wellIdentifier.getWellPosition().equals(wellPosition))
            {
                return wellIdentifier;
            }
        }
        throw new RuntimeException("Plate '" + augmentedPlateCode + "' has no well at "
                + wellPosition + ".");
    }

    /**
     * Fetches metadata of the image datasets for the specified <var>plate</var>.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Get the metadata of image datasets of plate P005 from space SPACE
     * imagesMetadata = OpenBISScreeningML.getImagesMetadata('/SPACE/P005');
     * % How many image datasets do we have? Usually there will be just one.
     * length(imagesMetadata)
     * % What is the number of tiles in the first image dataset?
     * imagesMetadata(1, 3)
     * </pre>
     * 
     * @param augmentedPlateCode The augmented plate code.
     * @return <code>{ images width, images height, number of tiles in the well, 
     *                 number of tiles rows, number of tiles columns, number of plate rows, number of plate columns }</code>
     *         .
     */
    public static Object[][] getImagesMetadata(String augmentedPlateCode)
    {
        checkLoggedIn();
        final Plate plateId = getPlate(augmentedPlateCode);
        final List<ImageDatasetReference> imageDatasets = listRawImageDatasets(plateId);
        final List<ImageDatasetMetadata> metaList = openbis.listImageMetadata(imageDatasets);

        Object[][] result = new Object[metaList.size()][];
        for (int i = 0; i < metaList.size(); ++i)
        {
            ImageDatasetMetadata meta = metaList.get(i);
            ImageDatasetReference imageDatasetReference = imageDatasets.get(i);
            Geometry plateGeometry = imageDatasetReference.getPlateGeometry();
            result[i] =
                    new Object[]
                        { meta.getWidth(), meta.getHeight(), meta.getNumberOfTiles(),
                                meta.getTilesRows(), meta.getTilesCols(),
                                plateGeometry.getNumberOfRows(), plateGeometry.getNumberOfColumns() };
        }
        return result;
    }

    /**
     * Lists all channels measured in <var>experiment</var>.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Get the channels of experiment MYEXP in project PROJ of space SPACE
     * channels = OpenBISScreeningML.listChannels('/SPACE/PROJ/MYEXP');
     * % How many channels do we have?
     * length(channels)
     * % What is the name of channel 1?
     * channels(1)
     * </pre>
     * 
     * @param experiment The augmented code of the experiment to list the channels for
     * @return Each row contains information about one channel. Currently the only information
     *         available is the channel name.
     */
    public static Object[][] listChannels(String experiment)
    {
        checkLoggedIn();
        final List<Plate> experimentPlates = experimentToPlateMap.get(experiment);
        if (experimentPlates == null)
        {
            throw new RuntimeException("No experiment with that code found.");
        }
        if (experimentPlates.isEmpty())
        {
            return new Object[0][];
        }
        final List<ImageDatasetReference> imageDatasets =
                openbis.listRawImageDatasets(experimentPlates);
        if (imageDatasets.isEmpty())
        {
            return new Object[0][];
        }
        return extractChannels(imageDatasets.get(0));
    }

    private static List<String> getChannelCodes(final List<ImageDatasetMetadata> meta)
    {
        return meta.get(0).getChannelCodes();
    }

    /**
     * Lists all features computed for <var>experiment</var>.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Get the features of experiment MYEXP in project PROJ of space SPACE
     * features = OpenBISScreeningML.listFeatures('/SPACE/PROJ/MYEXP');
     * % How many features do we have?
     * length(features)
     * % What is the name of features 1?
     * features(1)
     * </pre>
     * 
     * @param experiment The augmented code of the experiment to list the features for
     * @return Each row contains information about one feature. Currently the only information
     *         available is the feature name.
     */
    public static Object[][] listFeatures(String experiment)
    {
        checkLoggedIn();
        final List<Plate> experimentPlates = experimentToPlateMap.get(experiment);
        if (experimentPlates == null)
        {
            throw new RuntimeException("No experiment with that code found.");
        }
        if (experimentPlates.isEmpty())
        {
            return new Object[0][];
        }
        final List<FeatureVectorDatasetReference> featureDatasets =
                openbis.listFeatureVectorDatasets(experimentPlates);
        if (featureDatasets.isEmpty())
        {
            return new Object[0][];
        }
        List<String> featureCodes = openbis.listAvailableFeatureCodes(featureDatasets);
        Collections.sort(featureCodes);
        Object[][] result = new Object[featureCodes.size()][1];
        for (int i = 0; i < result.length; ++i)
        {
            result[i][0] = featureCodes.get(i);
        }
        return result;
    }

    /**
     * Loads data sets for specified plate code. For each data set the path to the root of the data
     * set is returned. If it is possible the path points directly into the data set store. No data
     * is copied. Otherwise the data is retrieved from the data store server.<br>
     * If the same dataset is loaded for the second time in one session it will be immediately
     * returned from the local cache.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Load all data sets of plate P005 in space SPACE
     * dsinfo = OpenBISScreeningML.loadDataSets('/SPACE/P005', '.*', '')
     * % Get the data set codes
     * dsinfo(:,1)
     * % Get root path of first data set (assuming there is at least one)
     * dsginfo(1,2)
     * </pre>
     * 
     * @param augmentedPlateCode The augmented plate code.
     * @param dataSetTypeCodePattern only datasets of the type which matches the specified pattern
     *            will be returned. To fetch all datasets specify ".*".
     * @param overrideStoreRootPathOrNull A path, in the context of the local file system mounts, to
     *            the DSS' store root. If null, paths are returned in the context of the DSS' file
     *            system mounts.
     * @return Each row contains information about one data set:
     *         <p>
     *         <code>{ data set code, data set root path  }</code>
     */
    public static Object[][] loadDataSets(String augmentedPlateCode, String dataSetTypeCodePattern,
            String overrideStoreRootPathOrNull)
    {
        checkLoggedIn();
        Plate plateIdentifier = getPlate(augmentedPlateCode);

        List<IDataSetDss> dataSets = openbis.getDataSets(plateIdentifier, dataSetTypeCodePattern);
        Object[][] result = new Object[dataSets.size()][];
        try
        {
            for (int i = 0; i < dataSets.size(); i++)
            {
                IDataSetDss dataSet = dataSets.get(i);
                String code = dataSet.getCode();
                File file = new File(dataSetsDir, code);
                if (file.exists() == false)
                {
                    file =
                            dataSet.getLinkOrCopyOfContents(overrideStoreRootPathOrNull,
                                    dataSetsDir);
                }
                result[i] = new Object[]
                    { code, file.getPath() };
            }
            return result;
        } catch (Exception ex)
        {
            throw new RuntimeException("Loading data sets for plate '" + augmentedPlateCode
                    + "' failed: " + ex, ex);
        }
    }
    
    /**
     * Loads file/folder of specified data set and specified file/folder path inside the data set.
     * If it is possible the path points directly into the data set store. No data is copied.
     * Otherwise the data is retrieved from the data store server.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % List all data sets of plate P005 in space SPACE. The query is restricted to data sets
     * % of a type starting with HCS_IMAGE
     * files = OpenBISScreeningML.listDataSetsFiles('/SPACE/P005', 'HCS_IMAGE.*')
     * % Load from the first data set (assuming at least one data set found) the third file/folder 
     * % (assuming at least three files/folders)
     * file = OpenBISScreeningML.loadDataSetFile(files(1,1), files(1,2,3), '')
     * </pre>
     * 
     * @param dataSetCode The code of the data set.
     * @param pathInDataSet Path inside the data set pointing to the file/folder which should be
     *            down loaded. Use '/' if all files are requested.
     * @param overrideStoreRootPathOrNull A path, in the context of the local file system mounts, to
     *            the DSS' store root. If null, paths are returned in the context of the DSS' file
     *            system mounts.
     * @return path to the down loaded file/folder.
     */
    public static Object loadDataSetFile(String dataSetCode, String pathInDataSet, String overrideStoreRootPathOrNull)
    {
        checkLoggedIn();
        IDataSetDss dataSet = openbis.getDataSet(dataSetCode);
        return dataSet.getLinkOrCopyOfContent(overrideStoreRootPathOrNull, temporarySessionDir, pathInDataSet).toString();
    }
    
    /**
     * Lists all files of all data sets for specifies plate and data set type code matching
     * specified regular expression pattern. 
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % List all data sets of plate P005 in space SPACE. The query is restricted to data sets
     * % of a type starting with HCS_IMAGE
     * files = OpenBISScreeningML.listDataSetsFiles('/SPACE/P005', 'HCS_IMAGE.*')
     * % Codes of all found data sets
     * files(:,1)
     * % Code of third data set (assuming at least three data sets found)
     * files(3,1)
     * % Files of third data set (assuming at least three data sets found)
     * files(3,2,:)
     * </pre>
     * 
     * @param augmentedPlateCode The augmented plate code.
     * @param dataSetTypeCodePattern only data sets of the type which matches the specified pattern
     *            will be returned. To fetch all data sets specify ".*".
     * @return <code>{data set code, file/folder paths}</code>
     */
    public static Object[][][] listDataSetsFiles(String augmentedPlateCode, String dataSetTypeCodePattern)
    {
        checkLoggedIn();
        Plate plateIdentifier = getPlate(augmentedPlateCode);

        List<IDataSetDss> dataSets = openbis.getDataSets(plateIdentifier, dataSetTypeCodePattern);
        Object[][][] result = new Object[dataSets.size()][][];
        for (int i = 0; i < dataSets.size(); i++)
        {
            IDataSetDss dataSet = dataSets.get(i);
            FileInfoDssDTO[] fileInfos = dataSet.listFiles("/", true);
            String code = dataSet.getCode();
            result[i] = new Object[2][];
            result[i][0] = new Object[] {code};
            result[i][1] = new Object[fileInfos.length];
            for (int j = 0; j < fileInfos.length; j++)
            {
                FileInfoDssDTO fileInfo = fileInfos[j];
                result[i][1][j] = fileInfo.getPathInDataSet();
            }
        }
        return result;
    }

    /**
     * Uploads specified data set for specified plate. The data set code will be returned.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Upload data set /path/to/my-data-set with properties DESCRIPTION and NUMBER for 
     * % plate P005 in space SPACE
     * properties = {'DESCRIPTION' 'hello example'; 'NUMBER' 3.14}
     * datasetcode = OpenBISScreeningML.uploadDataSet('/SPACE/P005', '/path/to/my-data-set', 'HCS_IMAGE', properties)
     * </pre>
     * 
     * @param augmentedPlateCode The augmented plate code.
     * @param dataSetFilePath Path to the data set file/folder to be uploaded.
     * @param dataSetType Data set type.
     * @param dataSetProperties A two dimensional array where the first column contains the property
     *            codes and the second column the corresponding property values.
     */
    public static Object uploadDataSet(String augmentedPlateCode, String dataSetFilePath,
            String dataSetType, Object[][] dataSetProperties)
    {
        checkLoggedIn();
        Plate plateIdentifier = getPlate(augmentedPlateCode);
        File dataSetFile = new File(dataSetFilePath);
        if (dataSetFile.exists() == false)
        {
            throw new RuntimeException("Unknown data set file path '" + dataSetFilePath + "'.");
        }
        try
        {
            Map<String, String> map = createMap(dataSetProperties);
            IDataSetDss dataSet =
                    openbis.putDataSet(plateIdentifier, dataSetFile, new NewDataSetMetadataDTO(
                            dataSetType, map));
            return dataSet.getCode();
        } catch (Exception ex)
        {
            throw new RuntimeException("Couldn't upload data set for plate '" + augmentedPlateCode
                    + "'.", ex);
        }
    }

    private static Map<String, String> createMap(Object[][] properties)
    {
        Map<String, String> map = new HashMap<String, String>();
        for (Object[] objects : properties)
        {
            if (objects.length == 2)
            {
                Object value = objects[1];
                map.put(objects[0].toString(), value == null ? null : value.toString());
            }
        }
        return map;
    }

    //
    // Images
    //

    /**
     * Loads the TIFF images for the given well location, all tiles and all channels and stores them
     * in temporary files. The temporary files will be removed automatically when the Java Virtual
     * Machine exits.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Load the images for all channels of well B10 of plate P005 in space SPACE
     * imginfo = OpenBISScreeningML.loadImages('/SPACE/P005', 2, 10)
     * % Get the plate-well descriptions of all locations
     * imginfo(2,:,3)
     * % Show the third image (assuming there are at least three images)
     * imtool(imginfo(1,3))
     * </pre>
     * 
     * @param plate The augmented plate code
     * @param row The row in the plate to get the images for
     * @param col The column in the plate to get the images for
     * @return <code>{ names of TIFF files, image annotation }</code>
     *         <p>
     *         Each of <code>names of TIFF files</code> and <code>image annotation</code> is a
     *         vector of length of the number of images.
     *         <p>
     *         <code>image annotation</code> contains
     *         <code>{ channel name, tile number, plate well description, 
     *         plate augmented code, plate perm id, plate space code, plate code, row, column, 
     *         experiment augmented code, experiment perm id, experiment space code, 
     *         experiment project code, experiment code, data set code }</code>
     */
    public static Object[][][] loadImages(String plate, int row, int col)
    {
        return loadImages(plate, row, col, (String[]) null);
    }

    /**
     * Loads the TIFF images for the given well location, tile number, and all channels and stores
     * them in temporary files. The temporary files will be removed automatically when the Java
     * Virtual Machine exits.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Load the images for all channels of well B10 and tile 3 of plate P005 in space SPACE
     * imginfo = OpenBISScreeningML.loadImages('/SPACE/P005', 2, 10, 3)
     * % Get the plate-well descriptions of all locations
     * imginfo(2,:,3)
     * % Show the third image (assuming there are at least three images)
     * imtool(imginfo(1,3))
     * </pre>
     * 
     * @param plate The augmented plate code
     * @param row The row in the plate to get the images for
     * @param col The column in the plate to get the images for
     * @param tile The tile number. Starts with 0.
     * @return <code>{ names of TIFF files, image annotation }</code>
     *         <p>
     *         Each of <code>names of TIFF files</code> and <code>image annotation</code> is a
     *         vector of length of the number of images.
     *         <p>
     *         <code>image annotation</code> contains
     *         <code>{ channel name, tile number, plate well description, 
     *         plate augmented code, plate perm id, plate space code, plate code, row, column, 
     *         experiment augmented code, experiment perm id, experiment space code, 
     *         experiment project code, experiment code, data set code }</code>
     */
    public static Object[][][] loadImages(String plate, int row, int col, int tile)
    {
        return loadImages(plate, row, col, tile, (String[]) null);
    }

    /**
     * Loads the TIFF images for the given well location, list of channels, and all tiles and stores
     * them in temporary files. The temporary files will be removed automatically when the Java
     * Virtual Machine exits.
     * <p>
     * Matlab example:
     * 
     * <pre>
     * % Load the images for channel DAPI of well H10 of plate P005 in space SPACE
     * imginfo=OpenBISScreeningML.loadImages('/SPACE/P005', 8, 10, 'DAPI')
     * % Get the channel names and tile numbers of all locations
     * imginfo(2,:,1:2)
     * % Show the second image (assuming there are at least two images)
     * imtool(imginfo(1,2))
     * </pre>
     * 
     * @param plate The augmented plate code
     * @param row The row in the plate to get the images for
     * @param col The column in the plate to get the images for
     * @param channels The names of the channels to get the images for
     * @return <code>{ names of TIFF files, image annotation }</code>
     *         <p>
     *         Each of <code>names of TIFF files</code> and <code>image annotation</code> is a
     *         vector of length of the number of images.
     *         <p>
     *         <code>image annotation</code> contains
     *         <code>{ channel name, tile number, plate well description, 
     *         plate augmented code, plate perm id, plate space code, plate code, row, column, 
     *         experiment augmented code, experiment perm id, experiment space code, 
     *         experiment project code, experiment code, data set code }</code>
     */
    public static Object[][][] loadImages(String plate, int row, int col, String[] channels)
    {
        return loadRawImages(plate, row, col, channels, createAllTilesIterator());
    }

    /**
     * Has the same effect as {@link #loadImages(String, int, int, String[])}, but instead of
     * loading raw images loads their segmentation results if available.
     * 
     * @param objectNames The names of the segmentation objects to get the images for
     */
    public static Object[][][] loadSegmentationImages(String plate, int row, int col,