Skip to content
Snippets Groups Projects
Commit 2364b03e authored by tpylak's avatar tpylak
Browse files

SE-328, LMS-1966 jython dropbox handler (example) 

SVN: 19478
parent d5ec3940
No related branches found
No related tags found
No related merge requests found
Hide whitespace changes
Inline Side-by-side
Showing
with 325 additions and 166 deletions
bds/source/java/ch/systemsx/cisd/bds/hcs/Location.java
+ 1
− 1
View file @ 2364b03e
......@@ -46,7 +46,7 @@ public final class Location
this.y = y;
}
public static final Location tryCreateLocationFromRowAndColumn(final int row, final int column)
public static final Location createLocationFromRowAndColumn(final int row, final int column)
{
return new Location(column, row);
}
......
datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/dto/DataSetInformation.java
+ 1
− 1
View file @ 2364b03e
......@@ -330,7 +330,7 @@ public class DataSetInformation implements Serializable
//
@Override
public final String toString()
public String toString()
{
return ToStringBuilder.reflectionToString(this,
ModifiedShortPrefixToStringStyle.MODIFIED_SHORT_PREFIX_STYLE);
......
screening/.classpath
+ 1
− 0
View file @ 2364b03e
......@@ -45,5 +45,6 @@
<classpathentry kind="lib" path="/libraries/fastutil/fastutil.jar" sourcepath="/libraries/fastutil/fastutil-src.zip"/>
<classpathentry kind="lib" path="/libraries/jline/jline.jar" sourcepath="/libraries/jline/src.zip"/>
<classpathentry kind="lib" path="/libraries/sybit-image-viewer/image-viewer.jar"/>
<classpathentry kind="lib" path="/libraries/jython/jython.jar" sourcepath="/libraries/jython/src.zip"/>
<classpathentry kind="output" path="targets/www/WEB-INF/classes"/>
</classpath>
screening/.project
+ 23
− 17
View file @ 2364b03e
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>screening</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
</natures>
</projectDescription>
<?xml version="1.0" encoding="UTF-8"?>
<projectDescription>
<name>screening</name>
<comment></comment>
<projects>
</projects>
<buildSpec>
<buildCommand>
<name>org.python.pydev.PyDevBuilder</name>
<arguments>
</arguments>
</buildCommand>
<buildCommand>
<name>org.eclipse.jdt.core.javabuilder</name>
<arguments>
</arguments>
</buildCommand>
</buildSpec>
<natures>
<nature>org.eclipse.jdt.core.javanature</nature>
<nature>org.python.pydev.pythonNature</nature>
</natures>
</projectDescription>
screening/.pydevproject 0 → 100644
+ 7
− 0
View file @ 2364b03e
<?xml version="1.0" encoding="UTF-8"?>
<?eclipse-pydev version="1.0"?>
<pydev_project>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property>
<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property>
</pydev_project>
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageFileExtractor.java
+ 35
− 15
View file @ 2364b03e
......@@ -40,8 +40,8 @@ import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.openbis.dss.etl.ImageFileExtractionResult.Channel;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescription;
......@@ -85,6 +85,9 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
// --------
// tiles geometry, e.g. 3x4 if the well is divided into 12 tiles (3 rows, 4 columns)
public static final String TILE_GEOMETRY_PROPERTY = "well_geometry";
// comma separated list of channel names, order matters
@Deprecated
public static final String CHANNEL_NAMES = "channel-names";
......@@ -113,7 +116,7 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
protected final TileMapper tileMapperOrNull;
protected final Geometry wellGeometry;
protected final Geometry tileGeometry;
protected AbstractImageFileExtractor(Properties properties, boolean skipChannelsWithoutImages)
{
......@@ -130,11 +133,11 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
* are not present in some datasets (exceptions: image overlays, test screens).
*/
protected AbstractImageFileExtractor(List<ChannelDescription> channelDescriptionsOrNull,
Geometry wellGeometry, boolean skipChannelsWithoutImages, Properties properties)
Geometry tileGeometry, boolean skipChannelsWithoutImages, Properties properties)
{
assert wellGeometry != null : "wel geometry is null";
assert tileGeometry != null : "wel geometry is null";
this.wellGeometry = wellGeometry;
this.tileGeometry = tileGeometry;
this.channelDescriptionsOrNull = channelDescriptionsOrNull;
this.channelColorComponentsOrNull = tryGetChannelComponents(properties);
......@@ -142,7 +145,14 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
checkChannelsAndColorComponents();
this.tileMapperOrNull =
TileMapper.tryCreate(properties.getProperty(TILE_MAPPING_PROPERTY), wellGeometry);
TileMapper.tryCreate(properties.getProperty(TILE_MAPPING_PROPERTY), tileGeometry);
}
protected final static Geometry getMandatoryTileGeometry(Properties properties)
{
String spotGeometryText =
PropertyUtils.getMandatoryProperty(properties, TILE_GEOMETRY_PROPERTY);
return Geometry.createFromString(spotGeometryText);
}
private static Geometry getWellGeometry(final Properties properties)
......@@ -231,7 +241,8 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
}
}
return new ImageFileExtractionResult(acquiredImages,
Collections.unmodifiableList(invalidFiles), getAllChannels(acquiredImages));
Collections.unmodifiableList(invalidFiles), getAllChannels(acquiredImages),
tileGeometry);
}
......@@ -356,8 +367,7 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
List<Channel> channels = new ArrayList<Channel>();
for (ChannelDescription channelDescription : channelDescriptions)
{
channels.add(new Channel(channelDescription.getCode(), null, null, channelDescription
.getLabel()));
channels.add(new Channel(channelDescription.getCode(), channelDescription.getLabel()));
}
return channels;
}
......@@ -437,7 +447,7 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
}
for (ChannelDescription channelDescription : channelDescriptionsOrNull)
{
if (channelDescription.getCode().equals(channelCode))
if (channelDescription.getCode().equalsIgnoreCase(channelCode))
{
return;
}
......@@ -447,7 +457,7 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
channelDescriptionsOrNull);
}
protected final static List<AcquiredSingleImage> createImagesWithNoColorComponent(
public final static List<AcquiredSingleImage> createImagesWithNoColorComponent(
ImageFileInfo imageInfo)
{
List<AcquiredSingleImage> images = new ArrayList<AcquiredSingleImage>();
......@@ -455,14 +465,24 @@ abstract public class AbstractImageFileExtractor implements IImageFileExtractor
return images;
}
protected final static AcquiredSingleImage createImage(ImageFileInfo imageInfo,
public final static AcquiredSingleImage createImage(ImageFileInfo imageInfo,
ColorComponent colorComponentOrNull)
{
RelativeImageReference relativeImageRef =
new RelativeImageReference(imageInfo.getImageRelativePath(), null,
colorComponentOrNull);
return new AcquiredSingleImage(imageInfo.tryGetWellLocation(), imageInfo.getTileLocation(),
imageInfo.getChannelCode(), imageInfo.tryGetTimepoint(), imageInfo.tryGetDepth(),
Location wellLoc = null;
if (imageInfo.hasWellLocation())
{
wellLoc =
Location.createLocationFromRowAndColumn(imageInfo.tryGetWellRow(),
imageInfo.tryGetWellColumn());
}
Location tileLoc =
Location.createLocationFromRowAndColumn(imageInfo.getTileRow(),
imageInfo.getTileColumn());
return new AcquiredSingleImage(wellLoc, tileLoc, imageInfo.getChannelCode(),
imageInfo.tryGetTimepoint(), imageInfo.tryGetDepth(),
imageInfo.tryGetSeriesNumber(), relativeImageRef);
}
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/AbstractImageStorageProcessor.java
+ 4
− 18
View file @ 2364b03e
......@@ -32,7 +32,6 @@ import org.apache.commons.io.FilenameUtils;
import org.apache.commons.lang.time.DurationFormatUtils;
import org.apache.log4j.Logger;
import ch.systemsx.cisd.bds.hcs.Geometry;
import ch.systemsx.cisd.common.collections.CollectionUtils;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.Status;
......@@ -117,9 +116,6 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
protected static final Logger notificationLog = LogFactory.getLogger(LogCategory.NOTIFY,
PlateStorageProcessor.class);
// tiles geometry, e.g. 3x4 if the well is divided into 12 tiles (3 rows, 4 columns)
private static final String SPOT_GEOMETRY_PROPERTY = "well_geometry";
private static final String GENERATE_THUMBNAILS_PROPERTY = "generate-thumbnails";
private final static String COMPRESS_THUMBNAILS_PROPERTY = "compress-thumbnails";
......@@ -167,8 +163,6 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
protected final IImageFileExtractor imageFileExtractor;
protected final Geometry spotGeometry;
// --- internal state -------------
private IImagingQueryDAO currentTransaction;
......@@ -177,14 +171,13 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
public AbstractImageStorageProcessor(final Properties properties)
{
this(getMandatorySpotGeometry(properties), tryCreateImageExtractor(properties), properties);
this(tryCreateImageExtractor(properties), properties);
}
protected AbstractImageStorageProcessor(Geometry spotGeometry,
IImageFileExtractor imageFileExtractor, Properties properties)
protected AbstractImageStorageProcessor(IImageFileExtractor imageFileExtractor,
Properties properties)
{
super(properties);
this.spotGeometry = spotGeometry;
this.imageFileExtractor = imageFileExtractor;
this.thumbnailMaxWidth =
PropertyUtils.getInt(properties, THUMBNAIL_MAX_WIDTH_PROPERTY,
......@@ -202,13 +195,6 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
this.currentTransaction = null;
}
private static Geometry getMandatorySpotGeometry(Properties properties)
{
String spotGeometryText =
PropertyUtils.getMandatoryProperty(properties, SPOT_GEOMETRY_PROPERTY);
return Geometry.createFromString(spotGeometryText);
}
private static IImageFileExtractor tryCreateImageExtractor(final Properties properties)
{
String fileExtractorClass = PropertyUtils.getProperty(properties, FILE_EXTRACTOR_PROPERTY);
......@@ -382,7 +368,7 @@ abstract class AbstractImageStorageProcessor extends AbstractStorageProcessor
final File incomingDataSetDirectory)
{
long extractionStart = System.currentTimeMillis();
final ImageFileExtractionResult result =
ImageFileExtractionResult result =
getImageFileExtractor(incomingDataSetDirectory).extract(incomingDataSetDirectory,
dataSetInformation);
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/FlexibleHCSImageFileExtractor.java
+ 1
− 1
View file @ 2364b03e
......@@ -19,8 +19,8 @@ package ch.systemsx.cisd.openbis.dss.etl;
import java.io.File;
import java.util.Properties;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.UnparsedImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
/**
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageDatasetUploader.java
+ 3
− 2
View file @ 2364b03e
......@@ -22,6 +22,7 @@ import java.util.List;
import ch.systemsx.cisd.openbis.dss.etl.ImagingDatabaseHelper.ExperimentWithChannelsAndContainer;
import ch.systemsx.cisd.openbis.dss.etl.ImagingDatabaseHelper.ImagingChannelsMap;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgDatasetDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgSpotDTO;
......@@ -33,7 +34,7 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgSp
public class HCSImageDatasetUploader extends AbstractImageDatasetUploader
{
public static void upload(IImagingQueryDAO dao, HCSImageDatasetInfo info,
List<AcquiredSingleImage> images, List<ImageFileExtractionResult.Channel> channels)
List<AcquiredSingleImage> images, List<Channel> channels)
{
new HCSImageDatasetUploader(dao).upload(info, images, channels);
}
......@@ -44,7 +45,7 @@ public class HCSImageDatasetUploader extends AbstractImageDatasetUploader
}
private void upload(HCSImageDatasetInfo info, List<AcquiredSingleImage> images,
List<ImageFileExtractionResult.Channel> channels)
List<Channel> channels)
{
long contId;
ImagingChannelsMap channelsMap = null;
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/HCSImageFileExtractor.java
+ 19
− 12
View file @ 2364b03e
......@@ -24,8 +24,8 @@ import org.apache.commons.lang.StringUtils;
import ch.systemsx.cisd.bds.hcs.Location;
import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.UnparsedImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.CodeAndLabelUtil;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
......@@ -84,7 +84,7 @@ public class HCSImageFileExtractor extends AbstractImageFileExtractor
Location tileLoc = tryGetTileLocation(tileNumber);
if (tileLoc == null)
{
tileLoc = Location.tryCreateLocationFromRowwisePosition(tileNumber, wellGeometry);
tileLoc = Location.tryCreateLocationFromRowwisePosition(tileNumber, tileGeometry);
}
return tileLoc;
}
......@@ -160,13 +160,6 @@ public class HCSImageFileExtractor extends AbstractImageFileExtractor
{
assert unparsedInfo != null;
Location wellLocation = tryGetWellLocation(unparsedInfo.getWellLocationToken());
if (wellLocation == null)
{
operationLog.info("Cannot extract well location from token "
+ unparsedInfo.getWellLocationToken());
return null;
}
Location tileLocation = tryGetTileLocation(unparsedInfo.getTileLocationToken());
if (tileLocation == null)
{
......@@ -175,13 +168,27 @@ public class HCSImageFileExtractor extends AbstractImageFileExtractor
return null;
}
String channelCode = CodeAndLabelUtil.normalize(unparsedInfo.getChannelToken());
String imageRelativePath = getRelativeImagePath(incomingDataSetDirectory, imageFile);
ImageFileInfo info =
new ImageFileInfo(channelCode, tileLocation.getY(), tileLocation.getX(),
imageRelativePath);
boolean ok = info.setWell(unparsedInfo.getWellLocationToken());
if (ok == false)
{
operationLog.info("Cannot extract well location from token "
+ unparsedInfo.getWellLocationToken());
return null;
}
Float timepointOrNull = tryAsFloat(unparsedInfo.getTimepointToken());
info.setTimepoint(timepointOrNull);
Float depthOrNull = tryAsFloat(unparsedInfo.getDepthToken());
info.setDepth(depthOrNull);
Integer seriesNumberOrNull = tryAsInt(unparsedInfo.getSeriesNumberToken());
String imageRelativePath = getRelativeImagePath(incomingDataSetDirectory, imageFile);
info.setSeriesNumber(seriesNumberOrNull);
return new ImageFileInfo(wellLocation, channelCode, tileLocation, imageRelativePath,
timepointOrNull, depthOrNull, seriesNumberOrNull);
return info;
}
}
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImageFileExtractionResult.java
+ 10
− 49
View file @ 2364b03e
......@@ -18,6 +18,9 @@ package ch.systemsx.cisd.openbis.dss.etl;
import java.io.File;
import java.util.List;
import ch.systemsx.cisd.bds.hcs.Geometry;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
/**
* Class which contains the image extraction process results.
*
......@@ -33,12 +36,16 @@ public final class ImageFileExtractionResult
private final List<Channel> channels;
/** How many tile rows and columns are there on each spot? */
private final Geometry tileGeometry;
public ImageFileExtractionResult(List<AcquiredSingleImage> images, List<File> invalidFiles,
List<Channel> channels)
List<Channel> channels, Geometry tileGeometry)
{
this.images = images;
this.invalidFiles = invalidFiles;
this.channels = channels;
this.tileGeometry = tileGeometry;
}
public List<AcquiredSingleImage> getImages()
......@@ -56,55 +63,9 @@ public final class ImageFileExtractionResult
return channels;
}
/**
* A channel in which the image has been acquired.
* <p>
* Each channel has its <code>code</code> which uniquely identifies it in one experiment or
* dataset.
* </p>
*
* @author Tomasz Pylak
*/
public static final class Channel
public Geometry getTileGeometry()
{
private final String code;
private final String label;
private final String description;
private final Integer wavelength;
public Channel(String code, String descriptionOrNull, Integer wavelengthOrNull, String label)
{
assert code != null : "code is null";
assert label != null : "label is null";
this.label = label;
this.code = code;
this.description = descriptionOrNull;
this.wavelength = wavelengthOrNull;
}
public String getCode()
{
return code;
}
public String tryGetDescription()
{
return description;
}
public Integer tryGetWavelength()
{
return wavelength;
}
public String getLabel()
{
return label;
}
return tileGeometry;
}
}
\ No newline at end of file
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/ImagingDatabaseHelper.java
+ 10
− 10
View file @ 2364b03e
......@@ -21,8 +21,8 @@ import java.util.List;
import java.util.Map;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.openbis.dss.etl.ImageFileExtractionResult.Channel;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgChannelDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgContainerDTO;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgExperimentDTO;
......@@ -47,7 +47,7 @@ public class ImagingDatabaseHelper
* Creates channels connected to the specified dataset id.
*/
public static ImagingChannelsMap createDatasetChannels(IImagingQueryDAO dao, long datasetId,
List<ImageFileExtractionResult.Channel> channels)
List<Channel> channels)
{
ChannelOwner channelOwner = ChannelOwner.createDataset(datasetId);
return new ImagingChannelsCreator(dao).getOrCreateChannelsMap(channelOwner, channels);
......@@ -93,7 +93,7 @@ public class ImagingDatabaseHelper
*/
public static ExperimentWithChannelsAndContainer getOrCreateExperimentWithChannelsAndContainer(
IImagingQueryDAO dao, HCSContainerDatasetInfo info,
List<ImageFileExtractionResult.Channel> channels)
List<Channel> channels)
{
ImagingDatabaseHelper helper = new ImagingDatabaseHelper(dao);
synchronized (IImagingQueryDAO.class)
......@@ -246,14 +246,14 @@ public class ImagingDatabaseHelper
}
public ImagingChannelsMap getOrCreateChannelsMap(ChannelOwner channelOwner,
List<ImageFileExtractionResult.Channel> channels)
List<Channel> channels)
{
Map<String, Long> map = getOrCreateChannels(channelOwner, channels);
return new ImagingChannelsMap(map);
}
private Map<String, Long> getOrCreateChannels(ChannelOwner channelOwner,
List<ImageFileExtractionResult.Channel> channels)
List<Channel> channels)
{
if (channelOwner.tryGetExperimentId() != null)
{
......@@ -278,7 +278,7 @@ public class ImagingDatabaseHelper
{
Map<String/* name */, ImgChannelDTO> existingChannels = asNameMap(allChannels);
Map<String, Long> map = new HashMap<String, Long>();
for (ImageFileExtractionResult.Channel channel : channels)
for (Channel channel : channels)
{
ImgChannelDTO channelDTO =
updateExperimentChannel(channel, expId, existingChannels);
......@@ -290,7 +290,7 @@ public class ImagingDatabaseHelper
private Map<String, Long> createChannels(ChannelOwner channelOwner, List<Channel> channels)
{
Map<String, Long> map = new HashMap<String, Long>();
for (ImageFileExtractionResult.Channel channel : channels)
for (Channel channel : channels)
{
ImgChannelDTO channelDTO = createChannel(channel, channelOwner);
addChannel(map, channelDTO);
......@@ -313,7 +313,7 @@ public class ImagingDatabaseHelper
return nameMap;
}
private ImgChannelDTO updateExperimentChannel(ImageFileExtractionResult.Channel channel,
private ImgChannelDTO updateExperimentChannel(Channel channel,
long expId, Map<String, ImgChannelDTO> existingChannels)
{
ImgChannelDTO channelDTO =
......@@ -358,7 +358,7 @@ public class ImagingDatabaseHelper
channelName, existingChannels.keySet());
}
private ImgChannelDTO createChannel(ImageFileExtractionResult.Channel channel,
private ImgChannelDTO createChannel(Channel channel,
ChannelOwner channelOwner)
{
ImgChannelDTO channelDTO = makeChannelDTO(channel, channelOwner);
......@@ -367,7 +367,7 @@ public class ImagingDatabaseHelper
return channelDTO;
}
private static ImgChannelDTO makeChannelDTO(ImageFileExtractionResult.Channel channel,
private static ImgChannelDTO makeChannelDTO(Channel channel,
ChannelOwner channelOwner)
{
return new ImgChannelDTO(channel.getCode(), channel.tryGetDescription(),
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyBlackboxSeriesStorageProcessor.java
+ 13
− 10
View file @ 2364b03e
......@@ -27,7 +27,7 @@ import ch.systemsx.cisd.bds.hcs.Geometry;
import ch.systemsx.cisd.bds.hcs.Location;
import ch.systemsx.cisd.common.mail.IMailClient;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescription;
......@@ -52,22 +52,21 @@ public class MicroscopyBlackboxSeriesStorageProcessor extends AbstractImageStora
private static final Location DEFAULT_TILE = new Location(1, 1);
private static final Geometry DEFAULT_WELL_GEOMETRY = new Geometry(1, 1);
private static final Geometry DEFAULT_TILE_GEOMETRY = new Geometry(1, 1);
private static final List<ChannelDescription> DEFAULT_CHANNELS = Arrays
.asList(new ChannelDescription(DEFAULT_CHANNEL_CODE, DEFAULT_CHANNEL_LABEL));
public MicroscopyBlackboxSeriesStorageProcessor(Properties properties)
{
super(DEFAULT_WELL_GEOMETRY, new BlackboxSeriesImageFileExtractor(
properties), properties);
super(new BlackboxSeriesImageFileExtractor(properties), properties);
}
private static class BlackboxSeriesImageFileExtractor extends AbstractImageFileExtractor
{
protected BlackboxSeriesImageFileExtractor(Properties properties)
{
super(DEFAULT_CHANNELS, DEFAULT_WELL_GEOMETRY, true, properties);
super(DEFAULT_CHANNELS, DEFAULT_TILE_GEOMETRY, true, properties);
}
@Override
......@@ -76,8 +75,10 @@ public class MicroscopyBlackboxSeriesStorageProcessor extends AbstractImageStora
{
String imageRelativePath = getRelativeImagePath(incomingDataSetDirectory, imageFile);
// we postpone assigning series numbers until all images are extracted
return new ImageFileInfo(null, DEFAULT_CHANNEL_CODE, DEFAULT_TILE, imageRelativePath,
null, null, null);
ImageFileInfo info =
new ImageFileInfo(DEFAULT_CHANNEL_CODE, DEFAULT_TILE.getY(),
DEFAULT_TILE.getX(), imageRelativePath);
return info;
}
}
......@@ -88,7 +89,8 @@ public class MicroscopyBlackboxSeriesStorageProcessor extends AbstractImageStora
List<AcquiredSingleImage> images = extractedImages.getImages();
setSeriesNumber(images);
MicroscopyImageDatasetInfo dataset =
createMicroscopyImageDatasetInfo(dataSetInformation, images);
createMicroscopyImageDatasetInfo(dataSetInformation, images,
extractedImages.getTileGeometry());
MicroscopyImageDatasetUploader.upload(dao, dataset, images, extractedImages.getChannels());
}
......@@ -120,11 +122,12 @@ public class MicroscopyBlackboxSeriesStorageProcessor extends AbstractImageStora
}
private MicroscopyImageDatasetInfo createMicroscopyImageDatasetInfo(
DataSetInformation dataSetInformation, List<AcquiredSingleImage> images)
DataSetInformation dataSetInformation, List<AcquiredSingleImage> images,
Geometry tileGeometry)
{
boolean hasImageSeries = hasImageSeries(images);
return new MicroscopyImageDatasetInfo(dataSetInformation.getDataSetCode(),
spotGeometry.getRows(), spotGeometry.getColumns(), hasImageSeries);
tileGeometry.getRows(), tileGeometry.getColumns(), hasImageSeries);
}
@Override
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyImageDatasetUploader.java
+ 2
− 2
View file @ 2364b03e
......@@ -18,9 +18,9 @@ package ch.systemsx.cisd.openbis.dss.etl;
import java.util.List;
import ch.systemsx.cisd.openbis.dss.etl.ImageFileExtractionResult.Channel;
import ch.systemsx.cisd.openbis.dss.etl.ImagingDatabaseHelper.ImagingChannelsMap;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgDatasetDTO;
/**
......@@ -31,7 +31,7 @@ import ch.systemsx.cisd.openbis.plugin.screening.shared.imaging.dataaccess.ImgDa
public class MicroscopyImageDatasetUploader extends AbstractImageDatasetUploader
{
public static void upload(IImagingQueryDAO dao, MicroscopyImageDatasetInfo dataset,
List<AcquiredSingleImage> images, List<ImageFileExtractionResult.Channel> channels)
List<AcquiredSingleImage> images, List<Channel> channels)
{
new MicroscopyImageDatasetUploader(dao).upload(dataset, images, channels);
}
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyImageFileExtractor.java
+ 11
− 4
View file @ 2364b03e
......@@ -21,8 +21,8 @@ import java.util.Properties;
import ch.systemsx.cisd.bds.hcs.Location;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.openbis.dss.etl.dto.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.UnparsedImageFileInfo;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
import ch.systemsx.cisd.openbis.dss.generic.shared.utils.CodeAndLabelUtil;
import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.SampleIdentifier;
......@@ -71,14 +71,21 @@ public class MicroscopyImageFileExtractor extends AbstractImageFileExtractor
}
String channelCode = CodeAndLabelUtil.normalize(unparsedInfo.getChannelToken());
String imageRelativePath = getRelativeImagePath(incomingDataSetDirectory, imageFile);
ImageFileInfo info =
new ImageFileInfo(channelCode, tileLocation.getY(), tileLocation.getX(),
imageRelativePath);
Float timepointOrNull = tryAsFloat(unparsedInfo.getTimepointToken());
info.setTimepoint(timepointOrNull);
Float depthOrNull = tryAsFloat(unparsedInfo.getDepthToken());
info.setDepth(depthOrNull);
Integer seriesNumberOrNull = tryAsInt(unparsedInfo.getSeriesNumberToken());
String imageRelativePath = getRelativeImagePath(incomingDataSetDirectory, imageFile);
info.setSeriesNumber(seriesNumberOrNull);
return info;
return new ImageFileInfo(null, channelCode, tileLocation, imageRelativePath,
timepointOrNull, depthOrNull, seriesNumberOrNull);
}
}
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/MicroscopyStorageProcessor.java
+ 6
− 3
View file @ 2364b03e
......@@ -20,6 +20,7 @@ import java.io.File;
import java.util.List;
import java.util.Properties;
import ch.systemsx.cisd.bds.hcs.Geometry;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.mail.IMailClient;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
......@@ -52,17 +53,19 @@ public class MicroscopyStorageProcessor extends AbstractImageStorageProcessor
{
List<AcquiredSingleImage> images = extractedImages.getImages();
MicroscopyImageDatasetInfo dataset =
createMicroscopyImageDatasetInfo(dataSetInformation, images);
createMicroscopyImageDatasetInfo(dataSetInformation, images,
extractedImages.getTileGeometry());
MicroscopyImageDatasetUploader.upload(dao, dataset, images, extractedImages.getChannels());
}
private MicroscopyImageDatasetInfo createMicroscopyImageDatasetInfo(
DataSetInformation dataSetInformation, List<AcquiredSingleImage> images)
DataSetInformation dataSetInformation, List<AcquiredSingleImage> images,
Geometry tileGeometry)
{
boolean hasImageSeries = hasImageSeries(images);
return new MicroscopyImageDatasetInfo(dataSetInformation.getDataSetCode(),
spotGeometry.getRows(), spotGeometry.getColumns(), hasImageSeries);
tileGeometry.getRows(), tileGeometry.getColumns(), hasImageSeries);
}
@Override
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/PlateStorageProcessor.java
+ 32
− 21
View file @ 2364b03e
......@@ -24,7 +24,7 @@ import java.util.Set;
import org.apache.commons.lang.StringUtils;
import ch.systemsx.cisd.bds.hcs.Channel;
import ch.systemsx.cisd.bds.hcs.Geometry;
import ch.systemsx.cisd.bds.hcs.Location;
import ch.systemsx.cisd.bds.storage.IFile;
import ch.systemsx.cisd.bds.storage.filesystem.NodeFactory;
......@@ -38,6 +38,7 @@ import ch.systemsx.cisd.common.utilities.PropertyUtils;
import ch.systemsx.cisd.etlserver.IHCSImageFileAccepter;
import ch.systemsx.cisd.openbis.dss.etl.HCSImageCheckList.FullLocation;
import ch.systemsx.cisd.openbis.dss.etl.dataaccess.IImagingQueryDAO;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Experiment;
import ch.systemsx.cisd.openbis.plugin.screening.shared.basic.dto.ChannelDescription;
......@@ -121,7 +122,8 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
// adapts old-style image extractor to the new one which is stateless
private static IImageFileExtractor adapt(
final ch.systemsx.cisd.etlserver.IHCSImageFileExtractor extractor,
final File imageFileRootDirectory, final List<ChannelDescription> descriptions)
final File imageFileRootDirectory, final List<ChannelDescription> descriptions,
final Geometry tileGeometry)
{
return new IImageFileExtractor()
{
......@@ -135,22 +137,25 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
extractor.process(
NodeFactory.createDirectoryNode(incomingDataSetDirectory),
dataSetInformation, accepter);
List<ImageFileExtractionResult.Channel> channels =
convert(originalResult.getChannels());
List<Channel> channels = convert(originalResult.getChannels());
return new ImageFileExtractionResult(accepter.getImages(),
asRelativePaths(originalResult.getInvalidFiles()), channels);
asRelativePaths(originalResult.getInvalidFiles()), channels,
tileGeometry);
}
private List<ImageFileExtractionResult.Channel> convert(Set<Channel> channels)
private List<Channel> convert(Set<ch.systemsx.cisd.bds.hcs.Channel> channels)
{
List<ImageFileExtractionResult.Channel> result =
new ArrayList<ImageFileExtractionResult.Channel>();
for (Channel channel : channels)
List<Channel> result = new ArrayList<Channel>();
for (ch.systemsx.cisd.bds.hcs.Channel channel : channels)
{
result.add(new ImageFileExtractionResult.Channel(getChannelCodeOrLabel(
extractChannelCodes(descriptions), channel.getCounter()), null,
channel.getWavelength(), getChannelCodeOrLabel(
extractChannelLabels(descriptions), channel.getCounter())));
String channelCode =
getChannelCodeOrLabel(extractChannelCodes(descriptions),
channel.getCounter());
String channelLabel =
getChannelCodeOrLabel(extractChannelLabels(descriptions),
channel.getCounter());
Channel convertedChannel = new Channel(channelCode, channelLabel);
result.add(convertedChannel);
}
return result;
}
......@@ -172,7 +177,8 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
File incomingDataSetDirectory, ImageFileExtractionResult extractionResult)
{
HCSImageCheckList imageCheckList =
createImageCheckList(dataSetInformation, extractionResult.getChannels());
createImageCheckList(dataSetInformation, extractionResult.getChannels(),
extractionResult.getTileGeometry());
checkImagesForDuplicates(extractionResult, imageCheckList);
if (extractionResult.getInvalidFiles().size() > 0)
{
......@@ -206,15 +212,16 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
}
private HCSImageCheckList createImageCheckList(DataSetInformation dataSetInformation,
List<ch.systemsx.cisd.openbis.dss.etl.ImageFileExtractionResult.Channel> channels)
List<ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel> channels,
Geometry tileGeometry)
{
PlateDimension plateGeometry = getPlateGeometry(dataSetInformation);
List<String> channelCodes = new ArrayList<String>();
for (ch.systemsx.cisd.openbis.dss.etl.ImageFileExtractionResult.Channel channel : channels)
for (ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.Channel channel : channels)
{
channelCodes.add(channel.getCode());
}
return new HCSImageCheckList(channelCodes, plateGeometry, spotGeometry);
return new HCSImageCheckList(channelCodes, plateGeometry, tileGeometry);
}
private void checkCompleteness(HCSImageCheckList imageCheckList,
......@@ -265,9 +272,10 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
{
List<ChannelDescription> channelDescriptions =
AbstractImageFileExtractor.extractChannelDescriptions(properties);
Geometry tileGeometry = AbstractImageFileExtractor.getMandatoryTileGeometry(properties);
extractor =
adapt(deprecatedImageFileExtractor, incomingDataSetDirectory,
channelDescriptions);
channelDescriptions, tileGeometry);
}
return extractor;
}
......@@ -301,18 +309,21 @@ public final class PlateStorageProcessor extends AbstractImageStorageProcessor
Experiment experiment = dataSetInformation.tryToGetExperiment();
assert experiment != null : "experiment is null";
List<AcquiredSingleImage> images = extractedImages.getImages();
HCSImageDatasetInfo info = createImageDatasetInfo(experiment, dataSetInformation, images);
HCSImageDatasetInfo info =
createImageDatasetInfo(experiment, dataSetInformation, images,
extractedImages.getTileGeometry());
HCSImageDatasetUploader.upload(dao, info, images, extractedImages.getChannels());
}
private HCSImageDatasetInfo createImageDatasetInfo(Experiment experiment,
DataSetInformation dataSetInformation, List<AcquiredSingleImage> acquiredImages)
DataSetInformation dataSetInformation, List<AcquiredSingleImage> acquiredImages,
Geometry tileGeometry)
{
HCSContainerDatasetInfo info =
HCSContainerDatasetInfo.createScreeningDatasetInfo(dataSetInformation);
boolean hasImageSeries = hasImageSeries(acquiredImages);
return new HCSImageDatasetInfo(info, storeChannelsOnExperimentLevel,
spotGeometry.getRows(), spotGeometry.getColumns(), hasImageSeries);
tileGeometry.getRows(), tileGeometry.getColumns(), hasImageSeries);
}
}
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/UnparsedImageFileInfo.java
+ 1
− 0
View file @ 2364b03e
package ch.systemsx.cisd.openbis.dss.etl.dto;
import ch.systemsx.cisd.common.utilities.AbstractHashable;
import ch.systemsx.cisd.openbis.dss.etl.dto.api.v1.ImageFileInfo;
/**
* Intermediate DTO containing tokens from which image info {@link ImageFileInfo} can be extracted
......
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/Channel.java 0 → 100644
+ 60
− 0
View file @ 2364b03e
package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
import ch.systemsx.cisd.common.utilities.AbstractHashable;
/**
* A channel in which the image has been acquired.
* <p>
* Each channel has its <code>code</code> which uniquely identifies it in one experiment or dataset.
* </p>
*
* @author Tomasz Pylak
*/
public final class Channel extends AbstractHashable
{
private final String code;
private final String label;
private String description;
private Integer wavelength;
public Channel(String code, String label)
{
assert code != null : "code is null";
assert label != null : "label is null";
this.label = label;
this.code = code;
}
public String getCode()
{
return code;
}
public String tryGetDescription()
{
return description;
}
public Integer tryGetWavelength()
{
return wavelength;
}
public String getLabel()
{
return label;
}
public void setDescription(String description)
{
this.description = description;
}
public void setWavelength(Integer wavelength)
{
this.wavelength = wavelength;
}
}
\ No newline at end of file
screening/source/java/ch/systemsx/cisd/openbis/dss/etl/dto/api/v1/ImageDataSetInformation.java 0 → 100644
+ 85
− 0
View file @ 2364b03e
/*
* Copyright 2011 ETH Zuerich, CISD
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.systemsx.cisd.openbis.dss.etl.dto.api.v1;
import java.util.List;
import ch.systemsx.cisd.common.collections.CollectionUtils;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
import ch.systemsx.cisd.openbis.generic.shared.IServer;
/**
* Extends {@link DataSetInformation} with information about images needed in HCS/Microscopy.
*
* @author Tomasz Pylak
*/
public class ImageDataSetInformation extends DataSetInformation
{
private static final long serialVersionUID = IServer.VERSION;
private List<ImageFileInfo> images;
private List<Channel> channels;
private int tileRowsNumber, tileColumnsNumber;
public int getTileRowsNumber()
{
return tileRowsNumber;
}
public int getTileColumnsNumber()
{
return tileColumnsNumber;
}
public void setTileGeometry(int tileRowsNumber, int tileColumnsNumber)
{
this.tileRowsNumber = tileRowsNumber;
this.tileColumnsNumber = tileColumnsNumber;
}
public List<ImageFileInfo> getImages()
{
return images;
}
public void setImages(List<ImageFileInfo> images)
{
this.images = images;
}
public List<Channel> getChannels()
{
return channels;
}
public void setChannels(List<Channel> channels)
{
this.channels = channels;
}
@Override
public String toString()
{
return "[ dataset code = " + getDataSetCode() + ", tile Rows Number: " + tileRowsNumber
+ ", tile Columns Number: " + tileColumnsNumber + ", channels: "
+ CollectionUtils.abbreviate(channels, -1) + ", images: "
+ CollectionUtils.abbreviate(images, 20);
}
}
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment