From 81890caf9d1e577ab9e682db351381e76c538856 Mon Sep 17 00:00:00 2001
From: felmer <felmer>
Date: Mon, 17 Nov 2014 10:23:44 +0000
Subject: [PATCH] SSDM-1075: BlastDatabaseCreationMaintenanceTask finished.
Tests added to BlastDatabaseCreationMaintenanceTaskTest. Removing whitespaces
from sequences.
SVN: 32838
---
.../BlastDatabaseCreationMaintenanceTask.java | 373 ++++++++++++------
.../etlserver/plugins/BlastUtilities.java | 34 ++
.../FastaFileBuilderForDataSetFiles.java | 2 +-
.../plugins/GenericFastaFileBuilder.java | 10 +-
...stDatabaseCreationMaintenanceTaskTest.java | 246 ++++++++++--
5 files changed, 506 insertions(+), 159 deletions(-)
create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastUtilities.java
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTask.java
index 86059e687f8..e8d584fc1ac 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTask.java
@@ -18,33 +18,37 @@ package ch.systemsx.cisd.etlserver.plugins;
import java.io.BufferedReader;
import java.io.File;
-import java.io.FilenameFilter;
+import java.io.FileFilter;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.Comparator;
import java.util.Date;
-import java.util.HashSet;
-import java.util.LinkedHashSet;
+import java.util.EnumMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Properties;
import java.util.Set;
import java.util.StringTokenizer;
-import java.util.TreeMap;
+import java.util.TreeSet;
import java.util.regex.Pattern;
import java.util.regex.PatternSyntaxException;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
+import org.apache.commons.lang.StringUtils;
import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.fasta.FastaUtilities;
import ch.systemsx.cisd.common.fasta.SequenceType;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
import ch.systemsx.cisd.common.logging.LogCategory;
@@ -53,6 +57,8 @@ import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
import ch.systemsx.cisd.common.process.ProcessExecutionHelper;
import ch.systemsx.cisd.common.process.ProcessResult;
import ch.systemsx.cisd.common.properties.PropertyUtils;
+import ch.systemsx.cisd.common.string.Template;
+import ch.systemsx.cisd.etlserver.plugins.GenericFastaFileBuilder.EntryType;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.generic.shared.IConfigProvider;
@@ -64,8 +70,6 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.IEntityInformationHolderWit
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.CodeWithRegistrationAndModificationDate;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.EntityKind;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityPropertiesHolder;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.IEntityProperty;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ListSampleCriteria;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
@@ -87,6 +91,12 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
private static final String DEFAULT_LAST_SEEN_DATA_SET_FILE = "last-seen-data-set-for-BLAST-database-creation";
private static final String DEFAULT_FILE_TYPES = ".fasta .fa .fsa .fastq";
+ private static final String ID_DELIM = "+";
+ private static final String DB_NAME_DELIM = "+";
+ private static final Template ID_TEMPLATE = new Template("${entityKind}" + ID_DELIM + "${permId}"
+ + ID_DELIM + "${propertyType}" + ID_DELIM + "${timestamp}");
+ private static final String TIMESTAMP_TEMPLATE = "yyyyMMddHHmmss";
+
private static final Logger operationLog =
LogFactory.getLogger(LogCategory.OPERATION, BlastDatabaseCreationMaintenanceTask.class);
private static final Logger machineLog =
@@ -105,23 +115,16 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
@Override
public void setUp(String pluginName, Properties properties)
{
- List<String> dataSetTypeRegexs = PropertyUtils.getMandatoryList(properties, DATASET_TYPES_PROPERTY);
- dataSetTypePatterns = new ArrayList<Pattern>();
- for (String regex : dataSetTypeRegexs)
- {
- try
- {
- dataSetTypePatterns.add(Pattern.compile(regex));
- } catch (PatternSyntaxException ex)
- {
- throw new ConfigurationFailureException("Property '" + DATASET_TYPES_PROPERTY
- + "' has invalid regular expression '" + regex + "': " + ex.getMessage());
- }
- }
+ dataSetTypePatterns = getDataSetTypePatterns(properties);
fileTypes = Arrays.asList(properties.getProperty(FILE_TYPES_PROPERTY, DEFAULT_FILE_TYPES).split(" +"));
operationLog.info("File types: " + fileTypes);
lastSeenDataSetFile = getFile(properties, LAST_SEEN_DATA_SET_FILE_PROPERTY, DEFAULT_LAST_SEEN_DATA_SET_FILE);
loaders = createLoaders(properties);
+ if (dataSetTypePatterns.isEmpty() && loaders.isEmpty())
+ {
+ throw new ConfigurationFailureException("At least one of the two properties have to be defined: "
+ + DATASET_TYPES_PROPERTY + ", " + ENTITY_SEQUENCE_PROPERTIES_PROPERTY);
+ }
setUpBlastDatabasesFolder(properties);
setUpBlastTempFolder(properties);
String blastToolDirectory = BlastUtils.getBLASTToolDirectory(properties);
@@ -136,6 +139,27 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
}
+ private List<Pattern> getDataSetTypePatterns(Properties properties)
+ {
+ List<Pattern> patterns = new ArrayList<Pattern>();
+ List<String> dataSetTypeRegexs = PropertyUtils.tryGetList(properties, DATASET_TYPES_PROPERTY);
+ if (dataSetTypeRegexs != null)
+ {
+ for (String regex : dataSetTypeRegexs)
+ {
+ try
+ {
+ patterns.add(Pattern.compile(regex));
+ } catch (PatternSyntaxException ex)
+ {
+ throw new ConfigurationFailureException("Property '" + DATASET_TYPES_PROPERTY
+ + "' has invalid regular expression '" + regex + "': " + ex.getMessage());
+ }
+ }
+ }
+ return patterns;
+ }
+
private List<Loader> createLoaders(Properties properties)
{
String property = properties.getProperty(ENTITY_SEQUENCE_PROPERTIES_PROPERTY);
@@ -224,20 +248,72 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
createBlastDatabasesForDataSets(service, virtualDatabases);
createBlastDatabasesForEntities(service, virtualDatabases);
+ Collection<VirtualDatabase> values = virtualDatabases.values();
+ for (VirtualDatabase virtualDatabase : values)
+ {
+ virtualDatabase.save();
+ }
}
-
+
private void createBlastDatabasesForEntities(IEncapsulatedOpenBISService service,
Map<SequenceType, VirtualDatabase> virtualDatabases)
{
+ DateFormat dateFormat = new SimpleDateFormat(TIMESTAMP_TEMPLATE);
for (Loader loader : loaders)
{
- List<Sequence> sequences = loader.load(service);
+ Map<SequenceType, Sequences> map = loader.load(service);
+ for (Entry<SequenceType, Sequences> entry : map.entrySet())
+ {
+ SequenceType sequenceType = entry.getKey();
+ VirtualDatabase virtualDatabase = virtualDatabases.get(sequenceType);
+ Sequences sequences = entry.getValue();
+ String baseName = loader.getDefinition() + DB_NAME_DELIM
+ + dateFormat.format(sequences.getLatestModificationDate());
+ if (databaseExist(baseName, sequenceType))
+ {
+ virtualDatabase.keepDatabase(baseName);
+ continue;
+ }
+ GenericFastaFileBuilder builder = new GenericFastaFileBuilder(tmpFolder, baseName);
+ for (Sequence sequence : sequences.getSequences())
+ {
+ Template template = ID_TEMPLATE.createFreshCopy();
+ template.bind("entityKind", loader.getEntityKind());
+ template.bind("permId", sequence.getPermId());
+ template.bind("propertyType", sequence.getPropertyType());
+ template.bind("timestamp", dateFormat.format(sequence.getModificationDate()));
+ String id = template.createText();
+ builder.startEntry(EntryType.FASTA, id, sequenceType);
+ builder.appendToSequence(sequence.getSequence());
+ }
+ createBlastDatabases(builder, virtualDatabases);
+ }
}
}
+ private boolean databaseExist(String baseName, SequenceType sequenceType)
+ {
+ String[] fileNames = blastDatabasesFolder.list();
+ if (fileNames != null)
+ {
+ for (String fileName : fileNames)
+ {
+ if (fileName.startsWith(BlastUtilities.createDatabaseName(baseName, sequenceType)))
+ {
+ return true;
+ }
+ }
+ }
+ return false;
+ }
+
private void createBlastDatabasesForDataSets(IEncapsulatedOpenBISService service,
Map<SequenceType, VirtualDatabase> virtualDatabases)
{
+ if (dataSetTypePatterns.isEmpty())
+ {
+ return;
+ }
IHierarchicalContentProvider contentProvider = getContentProvider();
List<AbstractExternalData> dataSets = getDataSets(service);
if (dataSets.isEmpty() == false)
@@ -261,44 +337,37 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
}
}
+ private boolean dataSetTypeMatches(AbstractExternalData dataSet)
+ {
+ String dataSetType = dataSet.getDataSetType().getCode();
+ for (Pattern pattern : dataSetTypePatterns)
+ {
+ if (pattern.matcher(dataSetType).matches())
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
private Map<SequenceType, VirtualDatabase> loadVirtualDatabases(IEncapsulatedOpenBISService service)
{
- Map<SequenceType, VirtualDatabase> virtualDatabases = new TreeMap<SequenceType, VirtualDatabase>();
- Set<String> dataSetCodes = new HashSet<String>();
+ Map<SequenceType, VirtualDatabase> virtualDatabases = new EnumMap<SequenceType, VirtualDatabase>(SequenceType.class);
for (SequenceType sequenceType : SequenceType.values())
{
VirtualDatabase virtualDatabase = new VirtualDatabase(blastDatabasesFolder, sequenceType);
- dataSetCodes.addAll(virtualDatabase.getDataSetCodes());
virtualDatabases.put(sequenceType, virtualDatabase);
}
- if (dataSetCodes.isEmpty() == false)
+ for (DeletedDataSet deletedDataSet : service.listDeletedDataSets(null, null))
{
- Set<String> deletedDataSetCodes = new HashSet<String>();
- for (DeletedDataSet deletedDataSet : service.listDeletedDataSets(null, null))
- {
- deletedDataSetCodes.add(deletedDataSet.getCode());
- }
for (VirtualDatabase virtualDatabase : virtualDatabases.values())
{
- virtualDatabase.removeDeletedDataSets(deletedDataSetCodes);
+ virtualDatabase.deleteDatabase(deletedDataSet.getCode());
}
}
return virtualDatabases;
}
- private boolean dataSetTypeMatches(AbstractExternalData dataSet)
- {
- String dataSetType = dataSet.getDataSetType().getCode();
- for (Pattern pattern : dataSetTypePatterns)
- {
- if (pattern.matcher(dataSetType).matches())
- {
- return true;
- }
- }
- return false;
- }
-
private void createBlastDatabase(AbstractExternalData dataSet, Map<SequenceType, VirtualDatabase> virtualDatabases,
IHierarchicalContentProvider contentProvider)
{
@@ -307,7 +376,13 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
IHierarchicalContent content = contentProvider.asContent(dataSet);
IHierarchicalContentNode rootNode = content.getRootNode();
handle(rootNode, builder);
+ createBlastDatabases(builder, virtualDatabases);
+ }
+
+ private void createBlastDatabases(GenericFastaFileBuilder builder, Map<SequenceType, VirtualDatabase> virtualDatabases)
+ {
builder.finish();
+ String baseName = builder.getBaseName();
SequenceType[] values = SequenceType.values();
for (SequenceType sequenceType : values)
{
@@ -324,8 +399,8 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
"-title", databaseName, "-out", databaseFile);
if (success == false)
{
- operationLog.error("Creation of BLAST database failed for data set '" + dataSetCode
- + "'. Temporary fasta file: " + fastaFile);
+ operationLog.error("Creation of BLAST database '" + databaseName
+ + "' failed. Temporary fasta file: " + fastaFile);
break;
}
File databaseSeqFile = new File(databaseFile + ".nsq");
@@ -334,8 +409,7 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
process(makembindex, "-iformat", "blastdb", "-input", databaseFile, "-old_style_index", "false");
}
VirtualDatabase virtualDatabase = virtualDatabases.get(sequenceType);
- virtualDatabase.addDataSet(dataSetCode);
- virtualDatabase.save();
+ virtualDatabase.addDatabase(baseName);
}
builder.cleanUp();
}
@@ -468,6 +542,7 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
private static final class Loader
{
+ private final String definition;
private final String entityKind;
private final EntityLoader entityLoader;
private final String entityType;
@@ -475,10 +550,11 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
Loader(String definition)
{
- String[] items = definition.split(":");
+ this.definition = definition;
+ String[] items = StringUtils.splitByWholeSeparator(definition, DB_NAME_DELIM);
if (items.length != 3)
{
- throw new IllegalArgumentException("Definition not on form <entity kind>:<entity type>:<property type>");
+ throw new IllegalArgumentException("Definition not in form <entity kind>+<entity type>+<property type>");
}
entityKind = items[0];
entityLoader = EntityLoader.valueOf(entityKind);
@@ -486,9 +562,19 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
this.propertyType = items[2];
}
- List<Sequence> load(IEncapsulatedOpenBISService service)
+ String getDefinition()
+ {
+ return definition;
+ }
+
+ String getEntityKind()
+ {
+ return entityKind;
+ }
+
+ Map<SequenceType, Sequences> load(IEncapsulatedOpenBISService service)
{
- List<Sequence> list = new ArrayList<Sequence>();
+ Map<SequenceType, Sequences> map = new EnumMap<SequenceType, Sequences>(SequenceType.class);
for (IEntityInformationHolderWithProperties entity : entityLoader.listEntities(service, entityType))
{
List<IEntityProperty> properties = entity.getProperties();
@@ -499,42 +585,84 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
String sequence = property.tryGetAsString();
if (sequence != null)
{
- list.add(new Sequence(entity, propertyType, sequence));
+ Sequence seq = new Sequence(entity, propertyType, sequence);
+ Sequences sequences = map.get(seq.getSequenceType());
+ if (sequences == null)
+ {
+ sequences = new Sequences();
+ map.put(seq.getSequenceType(), sequences);
+ }
+ sequences.addSequence(seq);
}
break;
}
}
}
- return list;
+ return map;
+ }
+ }
+
+ private static final class Sequences
+ {
+ private final List<Sequence> sequences = new ArrayList<Sequence>();
+ private Date latestModificationDate = new Date(0);
+
+ void addSequence(Sequence sequence)
+ {
+ sequences.add(sequence);
+ if (latestModificationDate.compareTo(sequence.getModificationDate()) < 0)
+ {
+ latestModificationDate = sequence.getModificationDate();
+ }
+ }
+
+ List<Sequence> getSequences()
+ {
+ return sequences;
+ }
+
+ Date getLatestModificationDate()
+ {
+ return latestModificationDate;
}
}
private static final class Sequence
{
- private final EntityKind entityKind;
private final String permId;
private final Date modificationDate;
private final String propertyType;
private final String sequence;
+ private final SequenceType sequenceType;
+ @SuppressWarnings("rawtypes")
Sequence(IEntityInformationHolderWithProperties entity, String propertyType, String sequence)
{
this.propertyType = propertyType;
- this.sequence = sequence;
- entityKind = entity.getEntityKind();
+ this.sequence = removeWhiteSpaces(sequence);
permId = entity.getPermId();
+ Date date = null;
if (entity instanceof CodeWithRegistrationAndModificationDate)
{
- modificationDate = ((CodeWithRegistrationAndModificationDate) entity).getModificationDate();
- } else
- {
- modificationDate = null;
+ date = ((CodeWithRegistrationAndModificationDate) entity).getModificationDate();
}
+ modificationDate = date == null ? new Date() : date;
+ sequenceType = FastaUtilities.determineSequenceType(this.sequence);
}
- EntityKind getEntityKind()
+ private static String removeWhiteSpaces(String sequence)
{
- return entityKind;
+ StringBuilder builder = new StringBuilder();
+ for (int i = 0; i < sequence.length(); i++)
+ {
+ char c = sequence.charAt(i);
+ if (Character.isWhitespace(c) == false)
+ {
+ builder.append(c);
+ }
+ }
+ String string = builder.toString();
+ return string;
}
String getPermId()
@@ -556,6 +684,11 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
{
return sequence;
}
+
+ SequenceType getSequenceType()
+ {
+ return sequenceType;
+ }
}
@@ -568,9 +701,9 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
String entityTypeCode)
{
ListSampleCriteria criteria = new ListSampleCriteria();
- SampleType sampleType = new SampleType();
- sampleType.setCode(entityTypeCode);
+ SampleType sampleType = service.getSampleType(entityTypeCode);
criteria.setSampleType(sampleType);
+ criteria.setIncludeSpace(true);
return service.listSamples(criteria);
}
};
@@ -586,14 +719,15 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
private final String dbtype;
private final String postfix;
private final File databaseFile;
- private final Set<String> dataSetCodes = new LinkedHashSet<String>();
+ private final Set<String> databasesToBeDeleted = new TreeSet<String>();
+ private final Set<String> databasesToBeAdded = new TreeSet<String>();
private final String virtualDatabaseFileType;
VirtualDatabase(File databaseFolder, SequenceType sequenceType)
{
this.databaseFolder = databaseFolder;
dbtype = sequenceType.toString().toLowerCase();
- postfix = "-" + dbtype;
+ postfix = BlastUtilities.createDatabaseName("", sequenceType);
virtualDatabaseFileType = sequenceType == SequenceType.NUCL ? ".nal" : ".pal";
databaseFile = new File(databaseFolder, "all-" + dbtype + virtualDatabaseFileType);
if (databaseFile.isFile())
@@ -608,69 +742,47 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
String token = tokenizer.nextToken();
if (token.endsWith(postfix))
{
- dataSetCodes.add(token.substring(0, token.length() - postfix.length()));
+ String name = token.substring(0, token.length() - postfix.length());
+ if (isEntityPropertyDatabase(name))
+ {
+ databasesToBeDeleted.add(name);
+ } else
+ {
+ databasesToBeAdded.add(name);
+ }
}
}
}
}
}
- Set<String> getDataSetCodes()
+ void keepDatabase(String baseName)
{
- return dataSetCodes;
+ databasesToBeAdded.add(baseName);
+ databasesToBeDeleted.remove(baseName);
}
- void addDataSet(String dataSetCode)
+ void deleteDatabase(String baseName)
{
- dataSetCodes.add(dataSetCode);
+ databasesToBeDeleted.add(baseName);
+ databasesToBeAdded.remove(baseName);
}
-
- void removeDeletedDataSets(Collection<String> deletedDataSets)
+
+ void addDatabase(String name)
{
- boolean dataChanged = false;
- for (String dataSetCode : deletedDataSets)
- {
- if (dataSetCodes.remove(dataSetCode))
- {
- dataChanged = true;
- final String databaseName = dataSetCode + postfix;
- File[] files = databaseFolder.listFiles(new FilenameFilter()
- {
- @Override
- public boolean accept(File dir, String name)
- {
- return name.startsWith(databaseName);
- }
- });
- if (files != null)
- {
- boolean success = true;
- for (File file : files)
- {
- if (FileUtilities.delete(file) == false)
- {
- operationLog.warn("File deletion failed: " + file);
- success = false;
- }
- }
- if (success)
- {
- operationLog.info("BLAST database " + databaseName + " successfully deleted.");
- }
- }
- }
- }
- if (dataChanged)
- {
- save();
- }
+ databasesToBeAdded.add(name);
+ }
+
+ private boolean isEntityPropertyDatabase(String name)
+ {
+ return name.contains(DB_NAME_DELIM);
}
void save()
{
File allDatabaseFile = new File(databaseFolder, "all-" + dbtype + virtualDatabaseFileType);
File newAllDatabaseFile = new File(databaseFolder, "all-" + dbtype + virtualDatabaseFileType + ".new");
- if (dataSetCodes.isEmpty())
+ if (databasesToBeAdded.isEmpty())
{
if (allDatabaseFile.exists())
{
@@ -687,13 +799,46 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
{
StringBuilder builder = new StringBuilder();
builder.append("TITLE all-" + dbtype + "\nDBLIST");
- for (String dataSetCode : dataSetCodes)
+ for (String database : databasesToBeAdded)
{
- builder.append(' ').append(dataSetCode).append(postfix);
+ builder.append(' ').append(database).append(postfix);
}
FileUtilities.writeToFile(newAllDatabaseFile, builder.toString());
newAllDatabaseFile.renameTo(allDatabaseFile);
}
+ deleteDatabasesToBeDeleted();
+ }
+
+ private void deleteDatabasesToBeDeleted()
+ {
+ for (String database : databasesToBeDeleted)
+ {
+ final String databaseName = database + postfix;
+ File[] files = databaseFolder.listFiles(new FileFilter()
+ {
+ @Override
+ public boolean accept(File file)
+ {
+ return file.getName().startsWith(databaseName);
+ }
+ });
+ if (files != null && files.length > 0)
+ {
+ boolean success = true;
+ for (File file : files)
+ {
+ if (FileUtilities.delete(file) == false)
+ {
+ operationLog.warn("File deletion failed: " + file);
+ success = false;
+ }
+ }
+ if (success)
+ {
+ operationLog.info("BLAST database " + databaseName + " successfully deleted.");
+ }
+ }
+ }
}
}
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastUtilities.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastUtilities.java
new file mode 100644
index 00000000000..57adf05d635
--- /dev/null
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastUtilities.java
@@ -0,0 +1,34 @@
+/*
+ * Copyright 2014 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import ch.systemsx.cisd.common.fasta.SequenceType;
+
+/**
+ *
+ *
+ * @author Franz-Josef Elmer
+ */
+public class BlastUtilities
+{
+ public static String createDatabaseName(String baseName, SequenceType seqType)
+ {
+ return baseName + "-" + seqType.toString().toLowerCase();
+ }
+
+
+}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FastaFileBuilderForDataSetFiles.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FastaFileBuilderForDataSetFiles.java
index 185767c89e7..a26cd7075d6 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FastaFileBuilderForDataSetFiles.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/FastaFileBuilderForDataSetFiles.java
@@ -57,7 +57,7 @@ class FastaFileBuilderForDataSetFiles extends GenericFastaFileBuilder
{
throw new IllegalStateException("File path not set [Data Set: " + dataSetCode + "].");
}
- startEntry(entryType, line.substring(1) + " " + idExtension);
+ startEntry(entryType, line.substring(1) + " " + idExtension, null);
} else
{
appendToSequence(line);
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/GenericFastaFileBuilder.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/GenericFastaFileBuilder.java
index 0beea7faf81..0358aea2600 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/GenericFastaFileBuilder.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/GenericFastaFileBuilder.java
@@ -85,6 +85,11 @@ class GenericFastaFileBuilder
this.baseName = baseName;
}
+ String getBaseName()
+ {
+ return baseName;
+ }
+
void appendToSequence(String line)
{
if (currentFastaEntry == null)
@@ -106,10 +111,11 @@ class GenericFastaFileBuilder
return "Unspecified entry";
}
- void startEntry(EntryType entryType, String id)
+ void startEntry(EntryType entryType, String id, SequenceType sequenceTypeOrNull)
{
writeFastaEntry();
currentFastaEntry = new FastaEntry(id);
+ currentFastaEntry.setSeqType(sequenceTypeOrNull);
currentEntryType = entryType;
}
@@ -191,7 +197,7 @@ class GenericFastaFileBuilder
private File getFastaFile(SequenceType seqType)
{
- return new File(tempFolder, baseName + "-" + seqType.toString().toLowerCase() + ".fa");
+ return new File(tempFolder, BlastUtilities.createDatabaseName(baseName, seqType) + ".fa");
}
}
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTaskTest.java
index dae04dc8639..a7afc1fcbf5 100644
--- a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTaskTest.java
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTaskTest.java
@@ -18,15 +18,19 @@ package ch.systemsx.cisd.etlserver.plugins;
import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.BLAST_TEMP_FOLDER_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.DATASET_TYPES_PROPERTY;
+import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.ENTITY_SEQUENCE_PROPERTIES_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.FILE_TYPES_PROPERTY;
import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.LAST_SEEN_DATA_SET_FILE_PROPERTY;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.Date;
import java.util.List;
import java.util.Properties;
import org.apache.log4j.Level;
+import org.hamcrest.BaseMatcher;
+import org.hamcrest.Description;
import org.jmock.Expectations;
import org.jmock.Mockery;
import org.testng.annotations.AfterMethod;
@@ -46,9 +50,13 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.utils.BlastUtils;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DeletedDataSet;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ListSampleCriteria;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Sample;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TrackingDataSetCriteria;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.ContainerDataSetBuilder;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.DataSetBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.SampleBuilder;
import de.schlichtherle.io.File;
@@ -148,6 +156,8 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
private File store;
+ private File blastDatabaseFolder;
+
@BeforeMethod
public void setUpTask()
{
@@ -159,7 +169,8 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
maintenanceTask = new MockMaintenanceTask(configProvider, service, contentProvider);
maintenanceTask.setProcessSuccesses(true);
store = new File(workingDirectory, "store");
- store.mkdirs();
+ blastDatabaseFolder = new File(store, BlastUtils.DEFAULT_BLAST_DATABASES_FOLDER);
+ blastDatabaseFolder.mkdirs();
context.checking(new Expectations()
{
{
@@ -176,7 +187,7 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
}
@Test
- public void testSetUpMissingDataSetTypesProperty()
+ public void testSetUpMissingDataSetTypesPropertyAndEntitySequencePropertiesProperty()
{
try
{
@@ -184,8 +195,8 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
fail("ConfigurationFailureException expected.");
} catch (ConfigurationFailureException ex)
{
- assertEquals("Given key '" + BlastDatabaseCreationMaintenanceTask.DATASET_TYPES_PROPERTY
- + "' not found in properties '[]'", ex.getMessage());
+ assertEquals("At least one of the two properties have to be defined: dataset-types, entity-sequence-properties",
+ ex.getMessage());
}
context.assertIsSatisfied();
}
@@ -217,8 +228,8 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
maintenanceTask.setUp("", properties);
assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa, .fastq]\n"
- + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
- + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.\n"
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabaseFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + blastDatabaseFolder + "/tmp' created.\n"
+ ERROR_PREFIX + "BLAST isn't installed or property '" + BlastUtils.BLAST_TOOLS_DIRECTORY_PROPERTY
+ "' hasn't been correctly specified.", logRecorder.getLogContent());
context.assertIsSatisfied();
@@ -233,8 +244,8 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
maintenanceTask.setUp("BLAST databases creation", properties);
assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa, .fastq]\n"
- + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
- + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.",
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabaseFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + blastDatabaseFolder + "/tmp' created.",
logRecorder.getLogContent());
assertEquals("makeblastdb -version\n", maintenanceTask.getCommands());
context.assertIsSatisfied();
@@ -261,33 +272,34 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
FileUtilities.writeToFile(new File(dataSetFolder1, "fasta1.fa"), ">2\nGATTACA\nGATTACA\n");
FileUtilities.writeToFile(new File(dataFolder, "fasta2.fastq"), "@3\nIAKKATA\n+\nznhjnxzx\n");
prepareContentProvider(ds3, dataSetFolder1);
+ prepareListsDeletedDataSets();
StringBuilder stringBuilder = new StringBuilder();
for (int i = 0; i < 11e5; i++)
{
stringBuilder.append("GATTACA".charAt(i % 7));
}
// Creating a big sequence dummy file in order to trigger index creation
- FileUtilities.writeToFile(new File(store, "blast-databases/DS-3-prot.nsq"), stringBuilder.toString());
+ FileUtilities.writeToFile(new File(blastDatabaseFolder, "DS-3-prot.nsq"), stringBuilder.toString());
maintenanceTask.execute();
assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa, .fastq]\n"
- + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
- + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.\n"
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabaseFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + blastDatabaseFolder + "/tmp' created.\n"
+ INFO_PREFIX + "Scan 4 data sets for creating BLAST databases.", logRecorder.getLogContent());
assertEquals("makeblastdb -version\n"
- + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS-3-nucl.fa"
- + " -dbtype nucl -title DS-3-nucl -out " + store.getAbsolutePath() + "/blast-databases/DS-3-nucl\n"
- + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS-3-prot.fa"
- + " -dbtype prot -title DS-3-prot -out " + store.getAbsolutePath() + "/blast-databases/DS-3-prot\n"
- + "makembindex -iformat blastdb -input " + store.getAbsolutePath() + "/blast-databases/DS-3-prot"
+ + "makeblastdb -in " + blastDatabaseFolder.getAbsolutePath() + "/tmp/DS-3-nucl.fa"
+ + " -dbtype nucl -title DS-3-nucl -out " + blastDatabaseFolder.getAbsolutePath() + "/DS-3-nucl\n"
+ + "makeblastdb -in " + blastDatabaseFolder.getAbsolutePath() + "/tmp/DS-3-prot.fa"
+ + " -dbtype prot -title DS-3-prot -out " + blastDatabaseFolder.getAbsolutePath() + "/DS-3-prot\n"
+ + "makembindex -iformat blastdb -input " + blastDatabaseFolder.getAbsolutePath() + "/DS-3-prot"
+ " -old_style_index false\n",
maintenanceTask.getCommands());
assertEquals("TITLE all-nucl\nDBLIST DS-3-nucl",
- FileUtilities.loadToString(new File(store, "blast-databases/all-nucl.nal")).trim());
+ FileUtilities.loadToString(new File(blastDatabaseFolder, "all-nucl.nal")).trim());
assertEquals("TITLE all-prot\nDBLIST DS-3-prot",
- FileUtilities.loadToString(new File(store, "blast-databases/all-prot.pal")).trim());
- assertEquals("[]", Arrays.asList(new File(store, "blast-databases/tmp").listFiles()).toString());
+ FileUtilities.loadToString(new File(blastDatabaseFolder, "all-prot.pal")).trim());
+ assertEquals("[]", Arrays.asList(new File(blastDatabaseFolder, "tmp").listFiles()).toString());
assertEquals(0L, lastSeenIdMatcher.recordedObject().getLastSeenDataSetId());
context.assertIsSatisfied();
}
@@ -317,27 +329,135 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
dataSetFolder3.mkdirs();
FileUtilities.writeToFile(new File(dataSetFolder3, "fasta.fasta"), ">3\nGATTACA\nGATTACA\nGATTACA\n");
prepareContentProvider(ds3, dataSetFolder3);
+ prepareListsDeletedDataSets();
maintenanceTask.execute();
assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa, .fastq]\n"
- + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
- + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.\n"
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabaseFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + blastDatabaseFolder + "/tmp' created.\n"
+ INFO_PREFIX + "Scan 3 data sets for creating BLAST databases.", logRecorder.getLogContent());
assertEquals("makeblastdb -version\n"
- + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS2-nucl.fa"
- + " -dbtype nucl -title DS2-nucl -out " + store.getAbsolutePath() + "/blast-databases/DS2-nucl\n"
- + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS3-nucl.fa"
- + " -dbtype nucl -title DS3-nucl -out " + store.getAbsolutePath() + "/blast-databases/DS3-nucl\n",
+ + "makeblastdb -in " + blastDatabaseFolder.getAbsolutePath() + "/tmp/DS2-nucl.fa"
+ + " -dbtype nucl -title DS2-nucl -out " + blastDatabaseFolder.getAbsolutePath() + "/DS2-nucl\n"
+ + "makeblastdb -in " + blastDatabaseFolder.getAbsolutePath() + "/tmp/DS3-nucl.fa"
+ + " -dbtype nucl -title DS3-nucl -out " + blastDatabaseFolder.getAbsolutePath() + "/DS3-nucl\n",
maintenanceTask.getCommands());
assertEquals("TITLE all-nucl\nDBLIST DS2-nucl DS3-nucl",
- FileUtilities.loadToString(new File(store, "blast-databases/all-nucl.nal")).trim());
- assertEquals(false, new File(store, "blast-databases/all-prot.nal").exists());
- assertEquals("[]", Arrays.asList(new File(store, "blast-databases/tmp").listFiles()).toString());
+ FileUtilities.loadToString(new File(blastDatabaseFolder, "all-nucl.nal")).trim());
+ assertEquals(false, new File(blastDatabaseFolder, "all-prot.nal").exists());
+ assertEquals("[]", Arrays.asList(new File(blastDatabaseFolder, "tmp").listFiles()).toString());
assertEquals(0L, lastSeenIdMatcher.recordedObject().getLastSeenDataSetId());
context.assertIsSatisfied();
}
+ @Test
+ public void testExecuteForSequencesInSampleProperties()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(ENTITY_SEQUENCE_PROPERTIES_PROPERTY, "SAMPLE+T+O");
+ SampleBuilder s1 = new SampleBuilder("/A/S1").permID("1").property("A", "123");
+ SampleBuilder s2 = new SampleBuilder("/A/S2").permID("2").property("O", "GATTACA")
+ .modificationDate(new Date(123456789));
+ SampleBuilder s3 = new SampleBuilder("/A/S3").permID("3").property("O", "CAGATAA")
+ .modificationDate(new Date(987654321));
+ prepareListSamples("T", s1, s2, s3);
+ prepareListsDeletedDataSets();
+ maintenanceTask.setUp("", properties);
+
+ maintenanceTask.execute();
+
+ assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa, .fastq]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabaseFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + blastDatabaseFolder + "/tmp' created.", logRecorder.getLogContent());
+ assertEquals("makeblastdb -version\nmakeblastdb -in "
+ + blastDatabaseFolder.getAbsolutePath() + "/tmp/SAMPLE+T+O+19700112112054-nucl.fa"
+ + " -dbtype nucl -title SAMPLE+T+O+19700112112054-nucl -out "
+ + blastDatabaseFolder.getAbsolutePath() + "/SAMPLE+T+O+19700112112054-nucl\n",
+ maintenanceTask.getCommands());
+ assertEquals("TITLE all-nucl\nDBLIST SAMPLE+T+O+19700112112054-nucl",
+ FileUtilities.loadToString(new File(blastDatabaseFolder, "all-nucl.nal")).trim());
+ assertEquals(false, new File(blastDatabaseFolder, "all-prot.nal").exists());
+ assertEquals("[]", Arrays.asList(new File(blastDatabaseFolder, "tmp").listFiles()).toString());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testExecuteForSequencesInTwoSamplePropertiesRemovingPreviousVersion()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(ENTITY_SEQUENCE_PROPERTIES_PROPERTY, "SAMPLE+T+O, SAMPLE+T1+Q");
+ SampleBuilder s1 = new SampleBuilder("/A/S1").permID("1").property("A", "123");
+ SampleBuilder s2 = new SampleBuilder("/A/S2").permID("2").property("O", "GATTACA")
+ .modificationDate(new Date(123456789));
+ SampleBuilder s3 = new SampleBuilder("/A/S3").permID("3").property("Q", "CAGATAA")
+ .modificationDate(new Date(987654321));
+ prepareListSamples("T", s1, s2);
+ prepareListSamples("T1", s3);
+ prepareListsDeletedDataSets();
+ File previousDatabaseFile = new File(blastDatabaseFolder, "SAMPLE+T+O+19700112112010-nucl.nhr");
+ FileUtilities.writeToFile(previousDatabaseFile, "abc");
+ FileUtilities.writeToFile(new File(blastDatabaseFolder, "all-nucl.nal"),
+ "TITLE all-nucl\nDBLIST SAMPLE+T+O+19700112112010-nucl");
+ maintenanceTask.setUp("", properties);
+
+ maintenanceTask.execute();
+
+ assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa, .fastq]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabaseFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + blastDatabaseFolder + "/tmp' created.\n"
+ + INFO_PREFIX + "BLAST database SAMPLE+T+O+19700112112010-nucl successfully deleted.",
+ logRecorder.getLogContent());
+ assertEquals("makeblastdb -version\nmakeblastdb -in "
+ + blastDatabaseFolder.getAbsolutePath() + "/tmp/SAMPLE+T+O+19700102111736-nucl.fa"
+ + " -dbtype nucl -title SAMPLE+T+O+19700102111736-nucl -out "
+ + blastDatabaseFolder.getAbsolutePath() + "/SAMPLE+T+O+19700102111736-nucl\n"
+ + "makeblastdb -in " + blastDatabaseFolder.getAbsolutePath()
+ + "/tmp/SAMPLE+T1+Q+19700112112054-nucl.fa"
+ + " -dbtype nucl -title SAMPLE+T1+Q+19700112112054-nucl -out "
+ + blastDatabaseFolder.getAbsolutePath() + "/SAMPLE+T1+Q+19700112112054-nucl\n",
+ maintenanceTask.getCommands());
+ assertEquals("TITLE all-nucl\nDBLIST SAMPLE+T+O+19700102111736-nucl SAMPLE+T1+Q+19700112112054-nucl",
+ FileUtilities.loadToString(new File(blastDatabaseFolder, "all-nucl.nal")).trim());
+ assertEquals(false, previousDatabaseFile.exists());
+ assertEquals(false, new File(blastDatabaseFolder, "all-prot.nal").exists());
+ assertEquals("[]", Arrays.asList(new File(blastDatabaseFolder, "tmp").listFiles()).toString());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testExecuteForSequencesInSamplePropertiesWithNoUpdateNeeded()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(ENTITY_SEQUENCE_PROPERTIES_PROPERTY, "SAMPLE+T+O");
+ SampleBuilder s1 = new SampleBuilder("/A/S1").permID("1").property("A", "123");
+ SampleBuilder s2 = new SampleBuilder("/A/S2").permID("2").property("O", "GATTACA")
+ .modificationDate(new Date(123456789));
+ SampleBuilder s3 = new SampleBuilder("/A/S3").permID("3").property("O", "CAGATAA")
+ .modificationDate(new Date(987654321));
+ prepareListSamples("T", s1, s2, s3);
+ prepareListsDeletedDataSets();
+ File previousDatabaseFile = new File(blastDatabaseFolder, "SAMPLE+T+O+19700112112054-nucl.nhr");
+ FileUtilities.writeToFile(previousDatabaseFile, "abc");
+ FileUtilities.writeToFile(new File(blastDatabaseFolder, "all-nucl.nal"),
+ "TITLE all-nucl\nDBLIST SAMPLE+T+O+19700112112054-nucl");
+ maintenanceTask.setUp("", properties);
+
+ maintenanceTask.execute();
+
+ assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa, .fastq]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabaseFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + blastDatabaseFolder + "/tmp' created.",
+ logRecorder.getLogContent());
+ assertEquals("makeblastdb -version\n", maintenanceTask.getCommands());
+ assertEquals("TITLE all-nucl\nDBLIST SAMPLE+T+O+19700112112054-nucl",
+ FileUtilities.loadToString(new File(blastDatabaseFolder, "all-nucl.nal")).trim());
+ assertEquals(true, previousDatabaseFile.exists());
+ assertEquals(false, new File(blastDatabaseFolder, "all-prot.nal").exists());
+ assertEquals("[]", Arrays.asList(new File(blastDatabaseFolder, "tmp").listFiles()).toString());
+ context.assertIsSatisfied();
+ }
+
@Test
public void testExecuteFaileds()
{
@@ -351,23 +471,24 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
dataSetFolder1.mkdirs();
FileUtilities.writeToFile(new File(dataSetFolder1, "fasta.fastq"), "@1\nGATTACA\n+\nznhjnxzx\n");
prepareContentProvider(ds1, dataSetFolder1);
+ prepareListsDeletedDataSets();
maintenanceTask.setProcessSuccesses(false);
maintenanceTask.execute();
assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa, .fastq]\n"
- + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
- + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.\n"
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabaseFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + blastDatabaseFolder + "/tmp' created.\n"
+ INFO_PREFIX + "Scan 1 data sets for creating BLAST databases.\n"
- + ERROR_PREFIX + "Creation of BLAST database failed for data set 'DS1'. Temporary fasta file: "
- + store + "/blast-databases/tmp/DS1-nucl.fa", logRecorder.getLogContent());
+ + ERROR_PREFIX + "Creation of BLAST database 'DS1-nucl' failed. Temporary fasta file: "
+ + blastDatabaseFolder + "/tmp/DS1-nucl.fa", logRecorder.getLogContent());
assertEquals("makeblastdb -version\n"
- + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS1-nucl.fa"
- + " -dbtype nucl -title DS1-nucl -out " + store.getAbsolutePath() + "/blast-databases/DS1-nucl\n",
+ + "makeblastdb -in " + blastDatabaseFolder.getAbsolutePath() + "/tmp/DS1-nucl.fa"
+ + " -dbtype nucl -title DS1-nucl -out " + blastDatabaseFolder.getAbsolutePath() + "/DS1-nucl\n",
maintenanceTask.getCommands());
- assertEquals(false, new File(store, "blast-databases/all-nucl.nal").exists());
- assertEquals(false, new File(store, "blast-databases/all-prot.nal").exists());
- assertEquals("[]", Arrays.asList(new File(store, "blast-databases/tmp").listFiles()).toString());
+ assertEquals(false, new File(blastDatabaseFolder, "all-nucl.nal").exists());
+ assertEquals(false, new File(blastDatabaseFolder, "all-prot.nal").exists());
+ assertEquals("[]", Arrays.asList(new File(blastDatabaseFolder, "tmp").listFiles()).toString());
assertEquals(0L, lastSeenIdMatcher.recordedObject().getLastSeenDataSetId());
context.assertIsSatisfied();
}
@@ -393,7 +514,10 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
dataSetFolder1.mkdirs();
FileUtilities.writeToFile(new File(dataSetFolder1, "fasta.txt"), ">1\nGATTACA\n");
prepareContentProvider(ds1, dataSetFolder1);
+ prepareListsDeletedDataSets();
maintenanceTask.execute();
+ File ds1DatabaseFileDummy = new File(blastDatabasesFolder, "DS1-nucl.nhr");
+ FileUtilities.writeToFile(ds1DatabaseFileDummy, "blabla");
prepareListsDeletedDataSets(ds1.getCode());
AbstractExternalData ds2 = new DataSetBuilder(12L).type("BLAST").code("DS2")
.status(DataSetArchivingStatus.AVAILABLE).getDataSet();
@@ -406,10 +530,8 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
+ INFO_PREFIX + "BLAST databases folder: " + blastDatabasesFolder + "\n"
+ INFO_PREFIX + "Temp folder '" + tempFolder + "' created.\n"
+ INFO_PREFIX + "Scan 1 data sets for creating BLAST databases.\n"
- + INFO_PREFIX + "BLAST database DS1-nucl successfully deleted.\n"
- + INFO_PREFIX + "Virtual BLAST database file " + blastDatabasesFolder
- + "/all-nucl.nal deleted because it was empty.\n"
- + INFO_PREFIX + "Scan 1 data sets for creating BLAST databases.", logRecorder.getLogContent());
+ + INFO_PREFIX + "Scan 1 data sets for creating BLAST databases.\n"
+ + INFO_PREFIX + "BLAST database DS1-nucl successfully deleted.", logRecorder.getLogContent());
assertEquals("/usr/bin/blast/makeblastdb -version\n"
+ "/usr/bin/blast/makeblastdb -in " + tempFolder.getAbsolutePath() + "/DS1-nucl.fa"
+ " -dbtype nucl -title DS1-nucl -out " + blastDatabasesFolder.getAbsolutePath() + "/DS1-nucl\n"
@@ -418,7 +540,8 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
maintenanceTask.getCommands());
assertEquals("TITLE all-nucl\nDBLIST DS2-nucl",
FileUtilities.loadToString(new File(blastDatabasesFolder, "all-nucl.nal")).trim());
- assertEquals(false, new File(store, "blast-databases/all-prot.nal").exists());
+ assertEquals(false, new File(blastDatabaseFolder, "all-prot.nal").exists());
+ assertEquals(false, ds1DatabaseFileDummy.exists());
assertEquals("[]", Arrays.asList(tempFolder.listFiles()).toString());
assertEquals(11L, lastSeenIdMatcher.recordedObject().getLastSeenDataSetId());
assertEquals("12", FileUtilities.loadToString(lastSeenFile).trim());
@@ -452,6 +575,45 @@ public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystem
});
}
+ private void prepareListSamples(final String sampleType, final SampleBuilder... samples)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(service).getSampleType(sampleType);
+ final SampleType sType = new SampleType();
+ will(returnValue(sType));
+
+ one(service).listSamples(with(new BaseMatcher<ListSampleCriteria>()
+ {
+ @Override
+ public boolean matches(Object argument)
+ {
+ if (argument instanceof ListSampleCriteria)
+ {
+ ListSampleCriteria criteria = (ListSampleCriteria) argument;
+ assertSame(sType, criteria.getSampleType());
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public void describeTo(Description description)
+ {
+ description.appendText(sampleType);
+ }
+ }));
+ List<Sample> result = new ArrayList<Sample>();
+ for (SampleBuilder sample : samples)
+ {
+ result.add(sample.getSample());
+ }
+ will(returnValue(result));
+ }
+ });
+ }
+
private RecordingMatcher<TrackingDataSetCriteria> prepareListNewerDataSet(final AbstractExternalData... dataSets)
{
final RecordingMatcher<TrackingDataSetCriteria> recordingMatcher = new RecordingMatcher<TrackingDataSetCriteria>();
--
GitLab