From e890d2bbdb3218e40270916c67d95d9a0dbcfbcd Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Tue, 8 Jun 2010 12:29:03 +0000
Subject: [PATCH] LMS-1568 BDS migration
SVN: 16337
---
.../bdsmigration/BDSDataRemoverMigrator.java | 68 ++++
.../bdsmigration/BDSImagingDbUploader.java | 324 ++++++++++++++++++
.../BDSMigrationMaintananceTask.java | 227 ++++++++++++
.../dss/etl/bdsmigration/IBDSMigrator.java | 31 ++
.../OriginalDataRelocatorMigrator.java | 54 +++
.../ScreeningDatasetInfoExtractor.java | 131 +++++++
6 files changed, 835 insertions(+)
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSDataRemoverMigrator.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSImagingDbUploader.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSMigrationMaintananceTask.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/IBDSMigrator.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/OriginalDataRelocatorMigrator.java
create mode 100644 screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/ScreeningDatasetInfoExtractor.java
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSDataRemoverMigrator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSDataRemoverMigrator.java
new file mode 100644
index 00000000000..46de3964491
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSDataRemoverMigrator.java
@@ -0,0 +1,68 @@
+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.bdsmigration;
+
+import java.io.File;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+
+/**
+ * Removes unnecessary BDS data as a part of the migration.
+ *
+ * @author Tomasz Pylak
+ */
+class BDSDataRemoverMigrator implements IBDSMigrator
+{
+
+ public String getDescription()
+ {
+ return "removing unnecessary BDS data";
+ }
+
+ public boolean migrate(File dataset)
+ {
+ if (BDSMigrationMaintananceTask.tryGetOriginalDir(dataset) != null)
+ {
+ BDSMigrationMaintananceTask.logError(dataset, "original data has not been moved");
+ return false;
+ }
+ try
+ {
+ removeDir(dataset, BDSMigrationMaintananceTask.METADATA_DIR);
+ removeDir(dataset, BDSMigrationMaintananceTask.VERSION_DIR);
+ removeDir(dataset, BDSMigrationMaintananceTask.ANNOTATIONS_DIR);
+ removeDir(dataset, BDSMigrationMaintananceTask.DATA_DIR);
+ } catch (EnvironmentFailureException ex)
+ {
+ return false;
+ }
+ return true;
+ }
+
+ private void removeDir(File dataset, String relativeDirPath) throws EnvironmentFailureException
+ {
+ File dir = new File(dataset, relativeDirPath);
+ boolean ok = FileUtilities.deleteRecursively(dir);
+ if (ok == false)
+ {
+ String errorMsg = "Cannot delete the directory: " + dir.getAbsolutePath();
+ BDSMigrationMaintananceTask.operationLog.error(errorMsg);
+ throw new EnvironmentFailureException(errorMsg);
+ }
+ }
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSImagingDbUploader.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSImagingDbUploader.java
new file mode 100644
index 00000000000..55c68414a91
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSImagingDbUploader.java
@@ -0,0 +1,324 @@
+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.bdsmigration;
+
+import static ch.systemsx.cisd.openbis.dss.etl.bdsmigration.BDSMigrationMaintananceTask.DIR_SEP;
+import static ch.systemsx.cisd.openbis.dss.etl.bdsmigration.BDSMigrationMaintananceTask.METADATA_DIR;
+import static ch.systemsx.cisd.openbis.dss.etl.bdsmigration.BDSMigrationMaintananceTask.ORIGINAL_DIR;
+import static ch.systemsx.cisd.openbis.dss.etl.bdsmigration.BDSMigrationMaintananceTask.asNum;
+import static ch.systemsx.cisd.openbis.dss.etl.bdsmigration.BDSMigrationMaintananceTask.readLines;
+import static ch.systemsx.cisd.openbis.dss.etl.bdsmigration.BDSMigrationMaintananceTask.tryGetOriginalDir;
+
+import java.io.File;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Properties;
+import java.util.Set;
+
+import javax.sql.DataSource;
+
+import net.lemnik.eodsql.QueryTool;
+
+import org.apache.commons.lang.StringUtils;
+
+import ch.systemsx.cisd.bds.hcs.Location;
+import ch.systemsx.cisd.openbis.dss.etl.AcquiredPlateImage;
+import ch.systemsx.cisd.openbis.dss.etl.HCSDatasetUploader;
+import ch.systemsx.cisd.openbis.dss.etl.HCSImageFileExtractionResult;
+import ch.systemsx.cisd.openbis.dss.etl.RelativeImagePath;
+import ch.systemsx.cisd.openbis.dss.etl.ScreeningContainerDatasetInfo;
+import ch.systemsx.cisd.openbis.dss.etl.HCSImageFileExtractionResult.Channel;
+import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingUploadDAO;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+
+/**
+ * Uploads data to the imaging database.
+ *
+ * @author Tomasz Pylak
+ */
+class BDSImagingDbUploader
+{
+ private static IImagingUploadDAO createQuery(Properties properties)
+ {
+ DataSource dataSource = ServiceProvider.getDataSourceProvider().getDataSource(properties);
+ return QueryTool.getQuery(dataSource, IImagingUploadDAO.class);
+ }
+
+ public static IBDSMigrator createImagingDbUploaderMigrator(Properties properties,
+ final String[] channelNames)
+ {
+ final IImagingUploadDAO dao = createQuery(properties);
+ return new IBDSMigrator()
+ {
+ public String getDescription()
+ {
+ return "uploading data to the imaging database";
+ }
+
+ public boolean migrate(File dataset)
+ {
+ return BDSImagingDbUploader.migrateDataset(dataset, dao, channelNames);
+ }
+ };
+ }
+
+ private static boolean migrateDataset(File dataset, IImagingUploadDAO dao, String[] channelNames)
+ {
+ String originalDatasetDirName = tryGetOriginalDatasetDirName(dataset);
+ if (originalDatasetDirName == null)
+ {
+ return false;
+ }
+ return new BDSImagingDbUploader(dataset, dao, originalDatasetDirName, channelNames)
+ .migrate();
+ }
+
+ private final File dataset;
+
+ private final IImagingUploadDAO dao;
+
+ private final String originalDatasetDirName;
+
+ private final String[] channelNames;
+
+ private BDSImagingDbUploader(File dataset, IImagingUploadDAO dao,
+ String originalDatasetDirName, String[] channelNames)
+ {
+ this.dataset = dataset;
+ this.dao = dao;
+ this.originalDatasetDirName = originalDatasetDirName;
+ this.channelNames = channelNames;
+ }
+
+ private boolean migrate()
+ {
+ List<AcquiredPlateImage> images = tryExtractMappings();
+ if (images == null)
+ {
+ return false;
+ }
+
+ String relativeImagesDirectory = getRelativeImagesDirectory();
+
+ ScreeningContainerDatasetInfo info =
+ ScreeningDatasetInfoExtractor.tryCreateInfo(dataset, relativeImagesDirectory);
+ if (info == null)
+ {
+ return false;
+ }
+
+ Set<HCSImageFileExtractionResult.Channel> channels = extractChannels();
+
+ return storeInDatabase(images, info, channels);
+ }
+
+ private Set<Channel> extractChannels()
+ {
+ Set<Channel> channels = new HashSet<Channel>();
+ for (String channelName : channelNames)
+ {
+ channels.add(new Channel(channelName, null, null));
+ }
+ return channels;
+ }
+
+ private boolean storeInDatabase(List<AcquiredPlateImage> images,
+ ScreeningContainerDatasetInfo info, Set<HCSImageFileExtractionResult.Channel> channels)
+ {
+ try
+ {
+ HCSDatasetUploader.upload(dao, info, images, channels);
+ } catch (Exception ex)
+ {
+ logError("Uploading to the imaging db failed: " + ex.getMessage());
+ dao.rollback();
+ return false;
+ }
+ dao.commit();
+ return true;
+ }
+
+ private String getRelativeImagesDirectory()
+ {
+ return ORIGINAL_DIR + DIR_SEP + originalDatasetDirName;
+ }
+
+ private static String tryGetOriginalDatasetDirName(File dataset)
+ {
+ File originalDir = tryGetOriginalDir(dataset);
+ if (originalDir == null)
+ {
+ return null;
+ }
+ File[] files = originalDir.listFiles();
+ if (files.length != 1)
+ {
+ BDSMigrationMaintananceTask.logError(dataset, "Original directory '" + originalDir
+ + "' should contain exactly one file, but contains " + files.length + ": "
+ + files);
+ return null;
+ }
+ return files[0].getName();
+ }
+
+ private List<AcquiredPlateImage> tryExtractMappings()
+ {
+ File mappingFile = new File(dataset, METADATA_DIR + DIR_SEP + "standard_original_mapping");
+ if (mappingFile.isFile() == false)
+ {
+ logError("File '" + mappingFile + "' does not exist.");
+ return null;
+ }
+
+ try
+ {
+ List<String> lines = readLines(mappingFile);
+ return tryParseMappings(lines);
+ } catch (IOException ex)
+ {
+ logError("Error when reading mapping file '" + mappingFile + "': " + ex.getMessage());
+ return null;
+ }
+ }
+
+ private List<AcquiredPlateImage> tryParseMappings(List<String> lines)
+ {
+ List<AcquiredPlateImage> images = new ArrayList<AcquiredPlateImage>();
+ for (String line : lines)
+ {
+ AcquiredPlateImage mapping = tryParseMapping(line);
+ if (mapping != null)
+ {
+ images.add(mapping);
+ } else
+ {
+ return null;
+ }
+ }
+ return images;
+ }
+
+ private AcquiredPlateImage tryParseMapping(String line)
+ {
+ String[] tokens = StringUtils.split(line);
+ if (tokens.length != 3)
+ {
+ logError("Wrong number of tokens in the mapping line: " + line);
+ return null;
+ } else
+ {
+ try
+ {
+ return tryParseMappingLine(tokens[0], tokens[2]);
+ } catch (NumberFormatException ex)
+ {
+ logError("Incorrect format of mapping line: " + line + ". Cannot parse a number: "
+ + ex.getMessage());
+ return null;
+ }
+ }
+ }
+
+ // Example of standardPath: channel2/row1/column4/row2_column2.tiff
+ private AcquiredPlateImage tryParseMappingLine(String standardPath, String originalPath)
+ throws NumberFormatException
+ {
+ String[] pathTokens = standardPath.split("/");
+ if (pathTokens.length != 4)
+ {
+ logError("Wrong number of tokens in standard path: " + standardPath);
+ return null;
+ }
+ int channelNum = asNum(pathTokens[0], "channel");
+ int row = asNum(pathTokens[1], "row");
+ int col = asNum(pathTokens[2], "column");
+
+ String[] tileTokens = tryParseTileToken(pathTokens[3]);
+ if (tileTokens == null)
+ {
+ return null;
+ }
+ int tileRow = asNum(tileTokens[0], "row");
+ int tileCol = asNum(tileTokens[1], "column");
+
+ RelativeImagePath relativeImagePath = tryGetRelativeImagePath(originalPath);
+ if (relativeImagePath == null)
+ {
+ return null;
+ }
+ String channelName = tryGetChannelName(channelNum, standardPath);
+ if (channelName == null)
+ {
+ return null;
+ }
+ return new AcquiredPlateImage(new Location(col, row), new Location(tileCol, tileRow),
+ channelName, null, null, relativeImagePath);
+ }
+
+ // channelId - starts with 1
+ private String tryGetChannelName(int channelId, String standardPath)
+ {
+ if (channelNames.length < channelId)
+ {
+ logError("Name of the channel with the id " + channelId
+ + " has not been configured but is referenced in the path: " + standardPath
+ + ".");
+ return null;
+ }
+ return channelNames[channelId - 1];
+ }
+
+ private RelativeImagePath tryGetRelativeImagePath(String originalPath)
+ {
+ if (originalPath.startsWith(originalDatasetDirName) == false)
+ {
+ logError("Original path " + originalPath + " should start with "
+ + originalDatasetDirName);
+ return null;
+ }
+ String relativePath = originalPath.substring(originalPath.length());
+ return new RelativeImagePath(relativePath);
+ }
+
+ // tileFile - e.g. row2_column2.tiff
+ private String[] tryParseTileToken(String tileFile)
+ {
+ String tileDesc;
+ int dotIndex = tileFile.indexOf(".");
+ if (dotIndex != -1)
+ {
+ tileDesc = tileFile.substring(0, dotIndex);
+ } else
+ {
+ tileDesc = tileFile;
+ }
+ String[] tileTokens = tileDesc.split("_");
+ if (tileTokens.length != 2)
+ {
+ logError("Wrong number of tokens in tile file name: " + tileDesc);
+ return null;
+ }
+ return tileTokens;
+ }
+
+ private void logError(String reason)
+ {
+ BDSMigrationMaintananceTask.logError(dataset, reason);
+ }
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSMigrationMaintananceTask.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSMigrationMaintananceTask.java
new file mode 100644
index 00000000000..5a368f6cd08
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/BDSMigrationMaintananceTask.java
@@ -0,0 +1,227 @@
+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.bdsmigration;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.etlserver.IMaintenanceTask;
+import ch.systemsx.cisd.openbis.dss.etl.PlateStorageProcessor;
+
+/**
+ * Maintenance task which migrates all the BDS datasets to the imaging database.
+ *
+ * @author Tomasz Pylak
+ */
+public class BDSMigrationMaintananceTask implements IMaintenanceTask
+{
+ static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, BDSMigrationMaintananceTask.class);
+
+ static final String ANNOTATIONS_DIR = "annotations";
+
+ static final String METADATA_DIR = "metadata";
+
+ static final String DATA_DIR = "data";
+
+ static final String VERSION_DIR = "version";
+
+ static final String ORIGINAL_DIR = "original";
+
+ private static final String STORE_ROOT_PROPERTY = "storeRoot";
+
+ static final String DIR_SEP = "/";
+
+ private File storeRoot;
+
+ private String[] channelNames;
+
+ private Properties properties;
+
+ public void setUp(String pluginName, Properties properties)
+ {
+ String storeRootPath = properties.getProperty(STORE_ROOT_PROPERTY);
+ if (storeRootPath == null)
+ {
+ throw new EnvironmentFailureException(STORE_ROOT_PROPERTY + " property not specified.");
+ }
+ this.storeRoot = new File(storeRootPath);
+ if (storeRoot.isDirectory() == false)
+ {
+ throw new EnvironmentFailureException(storeRoot
+ + " does not exist or is not a directory.");
+ }
+ this.channelNames = PlateStorageProcessor.extractChannelNames(properties);
+ this.properties = properties;
+ }
+
+ public void execute()
+ {
+ IBDSMigrator imagingDbUploader =
+ BDSImagingDbUploader.createImagingDbUploaderMigrator(properties, channelNames);
+ IBDSMigrator[] migrators =
+ new IBDSMigrator[]
+ { imagingDbUploader, new OriginalDataRelocatorMigrator(),
+ new BDSDataRemoverMigrator() };
+ for (IBDSMigrator migrator : migrators)
+ {
+ boolean ok = migrateStore(migrator);
+ if (ok == false)
+ {
+ operationLog.error("Migration stopped at: " + migrator.getDescription());
+ return;
+ }
+ }
+ }
+
+ private boolean migrateStore(IBDSMigrator migrator)
+ {
+ File[] files = storeRoot.listFiles();
+ for (File file : files)
+ {
+ String name = file.getName();
+ if (name.equals("error") == false && name.equals("unidentified") == false)
+ {
+ boolean ok = migrateDatabaseInstance(file, migrator);
+ if (ok == false)
+ {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ private boolean migrateDatabaseInstance(File dbInstanceDir, IBDSMigrator migrator)
+ {
+ int successCounter = 0, failureCounter = 0;
+ for (File l1 : dbInstanceDir.listFiles())
+ {
+ for (File l2 : l1.listFiles())
+ {
+ for (File l3 : l2.listFiles())
+ {
+ for (File dataset : l3.listFiles())
+ {
+ boolean ok = migrateDataset(dataset, migrator);
+ if (ok)
+ {
+ successCounter++;
+ } else
+ {
+ failureCounter++;
+ }
+ }
+ }
+ }
+ }
+ logMigrationStats(migrator, successCounter, failureCounter);
+ return failureCounter == 0;
+ }
+
+ private void logMigrationStats(IBDSMigrator migrator, int successCounter, int failureCounter)
+ {
+ String desc = migrator.getDescription();
+ operationLog.info("Successful migration step '" + desc + "' of " + successCounter
+ + " datasets.");
+ operationLog.info("Unuccessful migration step '" + desc + "' of " + failureCounter
+ + " datasets.");
+ }
+
+ private boolean migrateDataset(File dataset, IBDSMigrator migrator)
+ {
+ if (isBDS(dataset))
+ {
+ boolean ok = migrator.migrate(dataset);
+ logMigrationFinished(ok, dataset, migrator.getDescription());
+ return ok;
+ } else
+ {
+ return true;
+ }
+ }
+
+ private static void logMigrationFinished(boolean ok, File dataset, String stepDescription)
+ {
+ String msg = "Migration step '" + stepDescription + "' of the dataset '" + dataset + "' ";
+ if (ok)
+ {
+ operationLog.info(msg + "succeeded.");
+ } else
+ {
+ operationLog.info(msg + "failed.");
+ }
+ }
+
+ static int asNum(String standardPathToken, String prefix) throws NumberFormatException
+ {
+ String number = standardPathToken.substring(prefix.length());
+ return Integer.parseInt(number);
+ }
+
+ @SuppressWarnings("unchecked")
+ static List<String> readLines(File mappingFile) throws IOException, FileNotFoundException
+ {
+ return IOUtils.readLines(new FileInputStream(mappingFile));
+ }
+
+ private static boolean isBDS(File dataset)
+ {
+ File[] files = dataset.listFiles();
+ return containsDir(files, VERSION_DIR) && containsDir(files, DATA_DIR)
+ && containsDir(files, METADATA_DIR) && containsDir(files, ANNOTATIONS_DIR);
+ }
+
+ private static boolean containsDir(File[] files, String dirName)
+ {
+ for (File file : files)
+ {
+ if (file.getName().equalsIgnoreCase(dirName))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ static File tryGetOriginalDir(File dataset)
+ {
+ File orgDir = new File(dataset, DATA_DIR + DIR_SEP + ORIGINAL_DIR);
+ if (orgDir.isDirectory() == false)
+ {
+ logError(dataset, "Original directory does not exist: " + orgDir);
+ return null;
+ }
+ return orgDir;
+ }
+
+ static void logError(File dataset, String reason)
+ {
+ operationLog.error("Cannot migrate dataset '" + dataset.getName() + "'. " + reason);
+ }
+
+}
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/IBDSMigrator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/IBDSMigrator.java
new file mode 100644
index 00000000000..67c7442645e
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/IBDSMigrator.java
@@ -0,0 +1,31 @@
+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.bdsmigration;
+
+import java.io.File;
+
+/**
+ * @author Tomasz Pylak
+ */
+interface IBDSMigrator
+{
+ /** migrates one BDS dataset */
+ boolean migrate(File dataset);
+
+ /** user-friendly description of the migrator */
+ String getDescription();
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/OriginalDataRelocatorMigrator.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/OriginalDataRelocatorMigrator.java
new file mode 100644
index 00000000000..3d93bafbdfd
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/OriginalDataRelocatorMigrator.java
@@ -0,0 +1,54 @@
+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.bdsmigration;
+
+import java.io.File;
+
+/**
+ * Second step of BDS migration, moves data from data/original to original.
+ *
+ * @author Tomasz Pylak
+ */
+class OriginalDataRelocatorMigrator implements IBDSMigrator
+{
+ public String getDescription()
+ {
+ return "moving data from data/original to original/";
+ }
+
+ public boolean migrate(File dataset)
+ {
+ File originalDir = BDSMigrationMaintananceTask.tryGetOriginalDir(dataset);
+ if (originalDir == null)
+ {
+ BDSMigrationMaintananceTask.operationLog.warn("No original data directory in dataset "
+ + dataset);
+ return false;
+ }
+ File destinationDir = new File(dataset, BDSMigrationMaintananceTask.ORIGINAL_DIR);
+ boolean ok = originalDir.renameTo(destinationDir);
+ if (ok == false)
+ {
+ BDSMigrationMaintananceTask.operationLog.error("Cannot move " + originalDir + " to "
+ + destinationDir);
+ return false;
+ } else
+ {
+ return true;
+ }
+ }
+}
\ No newline at end of file
diff --git a/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/ScreeningDatasetInfoExtractor.java b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/ScreeningDatasetInfoExtractor.java
new file mode 100644
index 00000000000..df7ef3c9de2
--- /dev/null
+++ b/screening/source/java/ch/systemsx/cisd/openbis/dss/etl/bdsmigration/ScreeningDatasetInfoExtractor.java
@@ -0,0 +1,131 @@
+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.etl.bdsmigration;
+
+import java.io.File;
+
+import ch.systemsx.cisd.common.filesystem.FileOperations;
+import ch.systemsx.cisd.openbis.dss.etl.ScreeningContainerDatasetInfo;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
+import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SpaceIdentifier;
+
+/**
+ * Extract the dataset metadata from BDS and by asking openBIS. Used to migrate BDS to imaging db.
+ *
+ * @author Tomasz Pylak
+ */
+class ScreeningDatasetInfoExtractor
+{
+ public static ScreeningContainerDatasetInfo tryCreateInfo(File dataset,
+ String relativeImagesDirectory)
+ {
+ Sample sample = tryGetSampleWithExperiment(dataset);
+ if (sample == null)
+ {
+ return null;
+ }
+ try
+ {
+ return createInfo(dataset, sample, relativeImagesDirectory);
+ } catch (Exception ex)
+ {
+ ex.printStackTrace();
+ BDSMigrationMaintananceTask.logError(dataset, "Unexpected exception: "
+ + ex.getMessage());
+ return null;
+ }
+ }
+
+ private static Sample tryGetSampleWithExperiment(File dataset)
+ {
+ IEncapsulatedOpenBISService openBISService = ServiceProvider.getOpenBISService();
+ SampleIdentifier sampleIdentifier = createSampleIdentifier(dataset);
+ Sample sample = openBISService.tryGetSampleWithExperiment(sampleIdentifier);
+ if (sample == null)
+ {
+ BDSMigrationMaintananceTask.logError(dataset, "Sample '" + sampleIdentifier
+ + "' cannot be found in openBIS");
+ }
+ return sample;
+ }
+
+ private static SampleIdentifier createSampleIdentifier(File dataset)
+ {
+ File sampleDir =
+ new File(dataset, BDSMigrationMaintananceTask.METADATA_DIR
+ + BDSMigrationMaintananceTask.DIR_SEP + "sample");
+ String databaseInstanceCode = contentAsString(new File(sampleDir, "instance_code"));
+ String spaceCode = contentAsString(new File(sampleDir, "space_code"));
+ String sampleCode = contentAsString(new File(sampleDir, "code"));
+
+ SpaceIdentifier spaceIdentifier = new SpaceIdentifier(databaseInstanceCode, spaceCode);
+ return new SampleIdentifier(spaceIdentifier, sampleCode);
+ }
+
+ private static ScreeningContainerDatasetInfo createInfo(File dataset, Sample sample,
+ String relativeImagesDirectory)
+ {
+ int rows = extractGeometryDim(dataset, "plate_geometry", "rows");
+ int columns = extractGeometryDim(dataset, "plate_geometry", "columns");
+ int tileRows = extractGeometryDim(dataset, "well_geometry", "rows");
+ int tileColumns = extractGeometryDim(dataset, "well_geometry", "columns");
+
+ ScreeningContainerDatasetInfo info = new ScreeningContainerDatasetInfo();
+ info.setContainerRows(rows);
+ info.setContainerColumns(columns);
+ info.setTileRows(tileRows);
+ info.setTileColumns(tileColumns);
+
+ info.setRelativeImagesDirectory(relativeImagesDirectory);
+
+ info.setDatasetPermId(extractDatasetPermId(dataset));
+ info.setContainerPermId(sample.getPermId());
+ info.setExperimentPermId(sample.getExperiment().getPermId());
+
+ return info;
+ }
+
+ private static int extractGeometryDim(File dataset, String geometryName, String fieldName)
+ {
+ File parentDir =
+ new File(BDSMigrationMaintananceTask.METADATA_DIR, "parameters"
+ + BDSMigrationMaintananceTask.DIR_SEP + geometryName);
+ return contentAsNumber(new File(parentDir, fieldName));
+ }
+
+ private static String extractDatasetPermId(File dataset)
+ {
+ File file =
+ new File(dataset, BDSMigrationMaintananceTask.METADATA_DIR
+ + BDSMigrationMaintananceTask.DIR_SEP + "data_set"
+ + BDSMigrationMaintananceTask.DIR_SEP + "code");
+ return contentAsString(file);
+ }
+
+ private static int contentAsNumber(File file)
+ {
+ return Integer.parseInt(contentAsString(file));
+ }
+
+ private static String contentAsString(File file)
+ {
+ return FileOperations.getInstance().getContentAsString(file);
+ }
+}
\ No newline at end of file
--
GitLab