diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTask.java
index 5a3329ebe48ce98d5c8b10faca63363cb53fbc99..9066602c7e092adfdd828054a397b1eaf9981c31 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTask.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTask.java
@@ -16,21 +16,20 @@
package ch.systemsx.cisd.etlserver.plugins;
-import java.io.BufferedInputStream;
import java.io.BufferedReader;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.io.InputStreamReader;
-import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.Comparator;
import java.util.List;
import java.util.Properties;
+import java.util.regex.Pattern;
+import java.util.regex.PatternSyntaxException;
-import org.apache.commons.io.FileUtils;
import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.IOUtils;
import org.apache.log4j.Logger;
@@ -44,6 +43,7 @@ import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.maintenance.IMaintenanceTask;
import ch.systemsx.cisd.common.process.ProcessExecutionHelper;
import ch.systemsx.cisd.common.process.ProcessResult;
+import ch.systemsx.cisd.common.properties.PropertyUtils;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContent;
import ch.systemsx.cisd.openbis.common.io.hierarchical_content.api.IHierarchicalContentNode;
import ch.systemsx.cisd.openbis.dss.generic.shared.IConfigProvider;
@@ -51,18 +51,20 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
-import ch.systemsx.cisd.openbis.generic.shared.basic.dto.PhysicalDataSet;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TrackingDataSetCriteria;
/**
- * This maintenance task creates a BLAST database for all files
+ * This maintenance task creates a BLAST database for all files with defined file types of data set with
+ * matching data set types.
*
* @author Franz-Josef Elmer
*/
public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
{
+ static final String DATASET_TYPES_PROPERTY = "dataset-types";
static final String BLAST_TOOLS_DIRECTORY_PROPERTY = "blast-tools-directory";
static final String BLAST_DATABASES_FOLDER_PROPERTY = "blast-databases-folder";
+ static final String BLAST_TEMP_FOLDER_PROPERTY = "blast-temp-folder";
static final String LAST_SEEN_DATA_SET_FILE_PROPERTY = "last-seen-data-set-file";
static final String FILE_TYPES_PROPERTY = "file-types";
@@ -77,6 +79,7 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
private File lastSeenDataSetFile;
+ private List<Pattern> dataSetTypePatterns;
private List<String> fileTypes;
private File blastDatabasesFolder;
private File tmpFolder;
@@ -86,17 +89,24 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
@Override
public void setUp(String pluginName, Properties properties)
{
+ List<String> dataSetTypeRegexs = PropertyUtils.getMandatoryList(properties, DATASET_TYPES_PROPERTY);
+ dataSetTypePatterns = new ArrayList<Pattern>();
+ for (String regex : dataSetTypeRegexs)
+ {
+ try
+ {
+ dataSetTypePatterns.add(Pattern.compile(regex));
+ } catch (PatternSyntaxException ex)
+ {
+ throw new ConfigurationFailureException("Property '" + DATASET_TYPES_PROPERTY
+ + "' has invalid regular expression '" + regex + "': " + ex.getMessage());
+ }
+ }
fileTypes = Arrays.asList(properties.getProperty(FILE_TYPES_PROPERTY, DEFAULT_FILE_TYPES).split(" +"));
operationLog.info("File types: " + fileTypes);
lastSeenDataSetFile = getFile(properties, LAST_SEEN_DATA_SET_FILE_PROPERTY, DEFAULT_LAST_SEEN_DATA_SET_FILE);
- blastDatabasesFolder = getFile(properties, BLAST_DATABASES_FOLDER_PROPERTY, DEFAULT_BLAST_DATABASES_FOLDER);
- operationLog.info("BLAST databases folder: " + blastDatabasesFolder);
- tmpFolder = new File(blastDatabasesFolder, "tmp");
- FileUtilities.deleteRecursively(tmpFolder);
- if (tmpFolder.mkdirs() == false)
- {
- throw new ConfigurationFailureException("Couldn't create folder '" + tmpFolder + "'.");
- }
+ setUpBlastDatabasesFolder(properties);
+ setUpBlastTempFolder(properties);
String blastToolDirectory = getBLASTToolDirectory(properties);
makeblastdb = blastToolDirectory + "makeblastdb";
if (process(makeblastdb, "-version") == false)
@@ -107,7 +117,53 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
makembindex = blastToolDirectory + "makembindex";
}
+
+ private void setUpBlastDatabasesFolder(Properties properties)
+ {
+ blastDatabasesFolder = getFile(properties, BLAST_DATABASES_FOLDER_PROPERTY, DEFAULT_BLAST_DATABASES_FOLDER);
+ operationLog.info("BLAST databases folder: " + blastDatabasesFolder);
+ if (blastDatabasesFolder.exists())
+ {
+ if (blastDatabasesFolder.isFile())
+ {
+ throw new ConfigurationFailureException("BLAST databases folder '" + blastDatabasesFolder
+ + "' is an existing file.");
+ }
+ } else
+ {
+ if (blastDatabasesFolder.mkdirs() == false)
+ {
+ throw new ConfigurationFailureException("Couldn't create BLAST databases folder '"
+ + blastDatabasesFolder + "'.");
+ }
+ }
+ }
+ private void setUpBlastTempFolder(Properties properties)
+ {
+ String tempFolderProperty = properties.getProperty(BLAST_TEMP_FOLDER_PROPERTY);
+ if (tempFolderProperty == null)
+ {
+ tmpFolder = new File(blastDatabasesFolder, "tmp");
+ } else
+ {
+ tmpFolder = new File(tempFolderProperty);
+ }
+ if (tmpFolder.exists())
+ {
+ boolean success = FileUtilities.deleteRecursively(tmpFolder);
+ if (success == false)
+ {
+ operationLog.warn("Couldn't delete temp folder '" + tmpFolder + "'.");
+ }
+ }
+ if (tmpFolder.mkdirs() == false)
+ {
+ throw new ConfigurationFailureException("Couldn't create temp folder '" + tmpFolder + "'.");
+ }
+ operationLog.info("Temp folder '" + tmpFolder + "' created.");
+ }
+
private File getFile(Properties properties, String pathProperty, String defaultPath)
{
String path = properties.getProperty(pathProperty);
@@ -136,7 +192,7 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
}
for (AbstractExternalData dataSet : dataSets)
{
- if (dataSet.tryGetAsDataSet() != null && dataSet.isAvailable())
+ if (dataSet.tryGetAsDataSet() != null && dataSet.isAvailable() && dataSetTypeMatches(dataSet))
{
try
{
@@ -150,6 +206,19 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
updateLastSeenEventId(dataSet.getId());
}
}
+
+ private boolean dataSetTypeMatches(AbstractExternalData dataSet)
+ {
+ String dataSetType = dataSet.getDataSetType().getCode();
+ for (Pattern pattern : dataSetTypePatterns)
+ {
+ if (pattern.matcher(dataSetType).matches())
+ {
+ return true;
+ }
+ }
+ return false;
+ }
private void createBlastDatabase(AbstractExternalData dataSet, IHierarchicalContentProvider contentProvider)
{
@@ -175,6 +244,8 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
"-title", databaseName, "-out", databaseFile);
if (success == false)
{
+ operationLog.error("Creation of BLAST database failed for data set '" + dataSetCode
+ + "'. Temporary fasta file: " + fastaFile);
break;
}
File databaseSeqFile = new File(databaseFile + ".nsq");
@@ -197,7 +268,7 @@ public class BlastDatabaseCreationMaintenanceTask implements IMaintenanceTask
return process(Arrays.asList(command));
}
- private boolean process(List<String> command)
+ boolean process(List<String> command)
{
ProcessResult processResult = ProcessExecutionHelper.run(command, operationLog, machineLog);
if (processResult.isOK())
diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTaskTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTaskTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..0adc1099d30fd5f4547c9515de12375317eaf873
--- /dev/null
+++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/BlastDatabaseCreationMaintenanceTaskTest.java
@@ -0,0 +1,447 @@
+/*
+ * Copyright 2014 ETH Zuerich, SIS
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.etlserver.plugins;
+
+import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.BLAST_DATABASES_FOLDER_PROPERTY;
+import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.BLAST_TEMP_FOLDER_PROPERTY;
+import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.BLAST_TOOLS_DIRECTORY_PROPERTY;
+import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.DATASET_TYPES_PROPERTY;
+import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.FILE_TYPES_PROPERTY;
+import static ch.systemsx.cisd.etlserver.plugins.BlastDatabaseCreationMaintenanceTask.LAST_SEEN_DATA_SET_FILE_PROPERTY;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Properties;
+
+import org.apache.log4j.Level;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.base.tests.AbstractFileSystemTestCase;
+import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
+import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.logging.BufferedAppender;
+import ch.systemsx.cisd.common.test.RecordingMatcher;
+import ch.systemsx.cisd.openbis.common.io.hierarchical_content.DefaultFileBasedHierarchicalContentFactory;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IConfigProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IHierarchicalContentProvider;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.AbstractExternalData;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetArchivingStatus;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.TrackingDataSetCriteria;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.ContainerDataSetBuilder;
+import ch.systemsx.cisd.openbis.generic.shared.basic.dto.builders.DataSetBuilder;
+
+import de.schlichtherle.io.File;
+
+/**
+ * @author Franz-Josef Elmer
+ */
+public class BlastDatabaseCreationMaintenanceTaskTest extends AbstractFileSystemTestCase
+{
+ private static final String INFO_PREFIX = "INFO OPERATION." + BlastDatabaseCreationMaintenanceTask.class.getSimpleName() + " - ";
+ private static final String ERROR_PREFIX = "ERROR OPERATION." + BlastDatabaseCreationMaintenanceTask.class.getSimpleName() + " - ";
+
+ private static final class MockMaintenanceTask extends BlastDatabaseCreationMaintenanceTask
+ {
+ private final List<List<String>> commands = new ArrayList<List<String>>();
+
+ private final IConfigProvider configProvider;
+
+ private final IEncapsulatedOpenBISService service;
+
+ private final IHierarchicalContentProvider contentProvider;
+
+ private boolean[] processSuccesses;
+
+ private int indexOfNextProcessSuccessValue;
+
+ MockMaintenanceTask(IConfigProvider configProvider, IEncapsulatedOpenBISService service,
+ IHierarchicalContentProvider contentProvider)
+ {
+ this.configProvider = configProvider;
+ this.service = service;
+ this.contentProvider = contentProvider;
+
+ }
+
+ void setProcessSuccesses(boolean... processSuccesses)
+ {
+ this.processSuccesses = processSuccesses;
+ indexOfNextProcessSuccessValue = 0;
+ }
+
+ @Override
+ boolean process(List<String> command)
+ {
+ commands.add(command);
+ return processSuccesses[indexOfNextProcessSuccessValue++ % processSuccesses.length];
+ }
+
+ String getCommands()
+ {
+ StringBuilder builder = new StringBuilder();
+ for (List<String> command : commands)
+ {
+ for (int i = 0; i < command.size(); i++)
+ {
+ if (i > 0)
+ {
+ builder.append(' ');
+ }
+ builder.append(command.get(i));
+ }
+ builder.append('\n');
+ }
+ return builder.toString();
+ }
+
+ @Override
+ IConfigProvider getConfigProvider()
+ {
+ return configProvider;
+ }
+
+ @Override
+ IEncapsulatedOpenBISService getOpenBISService()
+ {
+ return service;
+ }
+
+ @Override
+ IHierarchicalContentProvider getContentProvider()
+ {
+ return contentProvider;
+ }
+ }
+
+ private BufferedAppender logRecorder;
+
+ private Mockery context;
+
+ private IConfigProvider configProvider;
+
+ private IEncapsulatedOpenBISService service;
+
+ private IHierarchicalContentProvider contentProvider;
+
+ private MockMaintenanceTask maintenanceTask;
+
+ private File store;
+
+ @BeforeMethod
+ public void setUpTask()
+ {
+ logRecorder = new BufferedAppender("%-5p %c - %m%n", Level.INFO);
+ context = new Mockery();
+ configProvider = context.mock(IConfigProvider.class);
+ service = context.mock(IEncapsulatedOpenBISService.class);
+ contentProvider = context.mock(IHierarchicalContentProvider.class);
+ maintenanceTask = new MockMaintenanceTask(configProvider, service, contentProvider);
+ maintenanceTask.setProcessSuccesses(true);
+ store = new File(workingDirectory, "store");
+ store.mkdirs();
+ context.checking(new Expectations()
+ {
+ {
+ allowing(configProvider).getStoreRoot();
+ will(returnValue(store));
+ }
+ });
+ }
+
+ @AfterMethod
+ public void tearDown()
+ {
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testSetUpMissingDataSetTypesProperty()
+ {
+ try
+ {
+ maintenanceTask.setUp("", new Properties());
+ fail("ConfigurationFailureException expected.");
+ } catch (ConfigurationFailureException ex)
+ {
+ assertEquals("Given key '" + BlastDatabaseCreationMaintenanceTask.DATASET_TYPES_PROPERTY
+ + "' not found in properties '[]'", ex.getMessage());
+ }
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testSetUpInvalidDataSetTypeRegex()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(DATASET_TYPES_PROPERTY, "[a-z");
+ try
+ {
+ maintenanceTask.setUp("", properties);
+ fail("ConfigurationFailureException expected.");
+ } catch (ConfigurationFailureException ex)
+ {
+ assertEquals("Property '" + DATASET_TYPES_PROPERTY + "' has invalid regular expression '[A-Z': "
+ + "Unclosed character class near index 3\n[A-Z\n ^", ex.getMessage());
+ }
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testWrongBlastToolsFolder()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(DATASET_TYPES_PROPERTY, ".+");
+ maintenanceTask.setProcessSuccesses(false);
+
+ maintenanceTask.setUp("", properties);
+
+ assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
+ + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.\n"
+ + ERROR_PREFIX + "BLAST isn't installed or property '" + BLAST_TOOLS_DIRECTORY_PROPERTY
+ + "' hasn't been correctly specified.", logRecorder.getLogContent());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testMinimumSetUp()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(DATASET_TYPES_PROPERTY, ".+");
+
+ maintenanceTask.setUp("BLAST databases creation", properties);
+
+ assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
+ + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.",
+ logRecorder.getLogContent());
+ assertEquals("makeblastdb -version\n", maintenanceTask.getCommands());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testExecuteOneDataSetWithThreeFastaFiles()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(DATASET_TYPES_PROPERTY, "BLAST.*");
+ maintenanceTask.setUp("", properties);
+ AbstractExternalData ds1 = new ContainerDataSetBuilder(1L).type("BLAST_CONTAINER").code("DS-1").getContainerDataSet();
+ AbstractExternalData ds2 = new DataSetBuilder(2L).type("NOBLAST").code("DS-2")
+ .status(DataSetArchivingStatus.AVAILABLE).getDataSet();
+ AbstractExternalData ds3 = new DataSetBuilder(3L).type("BLAST").code("DS-3")
+ .status(DataSetArchivingStatus.AVAILABLE).getDataSet();
+ AbstractExternalData ds4 = new DataSetBuilder(3L).type("BLAST").code("DS-4")
+ .status(DataSetArchivingStatus.ARCHIVED).getDataSet();
+ RecordingMatcher<TrackingDataSetCriteria> lastSeenIdMatcher = prepareListNewerDataSet(ds4, ds2, ds3, ds1);
+ File dataSetFolder1 = new File(workingDirectory, "data-set-example1");
+ File dataFolder = new File(dataSetFolder1, "data");
+ dataFolder.mkdirs();
+ FileUtilities.writeToFile(new File(dataSetFolder1, "fasta.txt"), ">1\nGATTACA\n");
+ FileUtilities.writeToFile(new File(dataSetFolder1, "fasta1.fa"), ">2\nGATTACA\nGATTACA\n");
+ FileUtilities.writeToFile(new File(dataFolder, "fasta2.fa"), ">3\nIJAB\n");
+ prepareContentProvider(ds3, dataSetFolder1);
+ StringBuilder stringBuilder = new StringBuilder();
+ for (int i = 0; i < 11e5; i++)
+ {
+ stringBuilder.append("GATTACA".charAt(i % 7));
+ }
+ // Creating a big sequence dummy file in order to trigger index creation
+ FileUtilities.writeToFile(new File(store, "blast-databases/DS-3-prot.nsq"), stringBuilder.toString());
+
+ maintenanceTask.execute();
+
+ assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
+ + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.\n"
+ + INFO_PREFIX + "Scan 4 data sets for creating BLAST databases.", logRecorder.getLogContent());
+ assertEquals("makeblastdb -version\n"
+ + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS-3-nucl.fa"
+ + " -dbtype nucl -title DS-3-nucl -out " + store.getAbsolutePath() + "/blast-databases/DS-3-nucl\n"
+ + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS-3-prot.fa"
+ + " -dbtype prot -title DS-3-prot -out " + store.getAbsolutePath() + "/blast-databases/DS-3-prot\n"
+ + "makembindex -iformat blastdb -input " + store.getAbsolutePath() + "/blast-databases/DS-3-prot"
+ + " -old_style_index false\n",
+ maintenanceTask.getCommands());
+ assertEquals("TITLE all-nucl\nDBLIST DS-3-nucl",
+ FileUtilities.loadToString(new File(store, "blast-databases/all-nucl.nal")).trim());
+ assertEquals("TITLE all-prot\nDBLIST DS-3-prot",
+ FileUtilities.loadToString(new File(store, "blast-databases/all-prot.nal")).trim());
+ assertEquals("[]", Arrays.asList(new File(store, "blast-databases/tmp").listFiles()).toString());
+ assertEquals(0L, lastSeenIdMatcher.recordedObject().getLastSeenDataSetId());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testExecuteThreeDataSetsWhereTwoHaveFastaFiles()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(DATASET_TYPES_PROPERTY, ".*");
+ maintenanceTask.setUp("", properties);
+ AbstractExternalData ds1 = new DataSetBuilder(1L).type("NOBLAST").code("DS1")
+ .status(DataSetArchivingStatus.AVAILABLE).getDataSet();
+ AbstractExternalData ds2 = new DataSetBuilder(2L).type("BLAST").code("DS2")
+ .status(DataSetArchivingStatus.AVAILABLE).getDataSet();
+ AbstractExternalData ds3 = new DataSetBuilder(3L).type("BLAST").code("DS3")
+ .status(DataSetArchivingStatus.AVAILABLE).getDataSet();
+ RecordingMatcher<TrackingDataSetCriteria> lastSeenIdMatcher = prepareListNewerDataSet(ds3, ds2, ds1);
+ File dataSetFolder1 = new File(workingDirectory, "data-set-example1");
+ dataSetFolder1.mkdirs();
+ FileUtilities.writeToFile(new File(dataSetFolder1, "fasta.txt"), ">1\nGATTACA\n");
+ prepareContentProvider(ds1, dataSetFolder1);
+ File dataSetFolder2 = new File(workingDirectory, "data-set-example2");
+ dataSetFolder2.mkdirs();
+ FileUtilities.writeToFile(new File(dataSetFolder2, "fasta.fa"), ">2\nGATTACA\nGATTACA\n");
+ prepareContentProvider(ds2, dataSetFolder2);
+ File dataSetFolder3 = new File(workingDirectory, "data-set-example3");
+ dataSetFolder3.mkdirs();
+ FileUtilities.writeToFile(new File(dataSetFolder3, "fasta.fasta"), ">3\nGATTACA\nGATTACA\nGATTACA\n");
+ prepareContentProvider(ds3, dataSetFolder3);
+
+ maintenanceTask.execute();
+
+ assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
+ + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.\n"
+ + INFO_PREFIX + "Scan 3 data sets for creating BLAST databases.", logRecorder.getLogContent());
+ assertEquals("makeblastdb -version\n"
+ + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS2-nucl.fa"
+ + " -dbtype nucl -title DS2-nucl -out " + store.getAbsolutePath() + "/blast-databases/DS2-nucl\n"
+ + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS3-nucl.fa"
+ + " -dbtype nucl -title DS3-nucl -out " + store.getAbsolutePath() + "/blast-databases/DS3-nucl\n",
+ maintenanceTask.getCommands());
+ assertEquals("TITLE all-nucl\nDBLIST DS2-nucl DS3-nucl",
+ FileUtilities.loadToString(new File(store, "blast-databases/all-nucl.nal")).trim());
+ assertEquals(false, new File(store, "blast-databases/all-prot.nal").exists());
+ assertEquals("[]", Arrays.asList(new File(store, "blast-databases/tmp").listFiles()).toString());
+ assertEquals(0L, lastSeenIdMatcher.recordedObject().getLastSeenDataSetId());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testExecuteFaileds()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(DATASET_TYPES_PROPERTY, ".*");
+ maintenanceTask.setUp("", properties);
+ AbstractExternalData ds1 = new DataSetBuilder(1L).type("BLAST").code("DS1")
+ .status(DataSetArchivingStatus.AVAILABLE).getDataSet();
+ RecordingMatcher<TrackingDataSetCriteria> lastSeenIdMatcher = prepareListNewerDataSet(ds1);
+ File dataSetFolder1 = new File(workingDirectory, "data-set-example1");
+ dataSetFolder1.mkdirs();
+ FileUtilities.writeToFile(new File(dataSetFolder1, "fasta.fa"), ">1\nGATTACA\n");
+ prepareContentProvider(ds1, dataSetFolder1);
+ maintenanceTask.setProcessSuccesses(false);
+
+ maintenanceTask.execute();
+
+ assertEquals(INFO_PREFIX + "File types: [.fasta, .fa, .fsa]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + store + "/blast-databases\n"
+ + INFO_PREFIX + "Temp folder '" + store + "/blast-databases/tmp' created.\n"
+ + INFO_PREFIX + "Scan 1 data sets for creating BLAST databases.\n"
+ + ERROR_PREFIX + "Creation of BLAST database failed for data set 'DS1'. Temporary fasta file: "
+ + store + "/blast-databases/tmp/DS1-nucl.fa", logRecorder.getLogContent());
+ assertEquals("makeblastdb -version\n"
+ + "makeblastdb -in " + store.getAbsolutePath() + "/blast-databases/tmp/DS1-nucl.fa"
+ + " -dbtype nucl -title DS1-nucl -out " + store.getAbsolutePath() + "/blast-databases/DS1-nucl\n",
+ maintenanceTask.getCommands());
+ assertEquals(false, new File(store, "blast-databases/all-nucl.nal").exists());
+ assertEquals(false, new File(store, "blast-databases/all-prot.nal").exists());
+ assertEquals("[]", Arrays.asList(new File(store, "blast-databases/tmp").listFiles()).toString());
+ assertEquals(0L, lastSeenIdMatcher.recordedObject().getLastSeenDataSetId());
+ context.assertIsSatisfied();
+ }
+
+ @Test
+ public void testExecuteTwiceWithNonDefaultParameters()
+ {
+ Properties properties = new Properties();
+ properties.setProperty(DATASET_TYPES_PROPERTY, ".*");
+ File blastDatabasesFolder = new File(workingDirectory, "blast-dbs");
+ properties.setProperty(BLAST_DATABASES_FOLDER_PROPERTY, blastDatabasesFolder.toString());
+ File tempFolder = new File(workingDirectory, "temp");
+ properties.setProperty(BLAST_TEMP_FOLDER_PROPERTY, tempFolder.toString());
+ properties.setProperty(BLAST_TOOLS_DIRECTORY_PROPERTY, "/usr/bin/blast");
+ properties.setProperty(FILE_TYPES_PROPERTY, ".txt .f");
+ File lastSeenFile = new File(workingDirectory, "last.txt");
+ properties.setProperty(LAST_SEEN_DATA_SET_FILE_PROPERTY, lastSeenFile.toString());
+ maintenanceTask.setUp("", properties);
+ AbstractExternalData ds1 = new DataSetBuilder(11L).type("BLAST").code("DS1")
+ .status(DataSetArchivingStatus.AVAILABLE).getDataSet();
+ RecordingMatcher<TrackingDataSetCriteria> lastSeenIdMatcher = prepareListNewerDataSet(ds1);
+ File dataSetFolder1 = new File(workingDirectory, "data-set-example1");
+ dataSetFolder1.mkdirs();
+ FileUtilities.writeToFile(new File(dataSetFolder1, "fasta.txt"), ">1\nGATTACA\n");
+ prepareContentProvider(ds1, dataSetFolder1);
+ maintenanceTask.execute();
+ AbstractExternalData ds2 = new DataSetBuilder(12L).type("BLAST").code("DS2")
+ .status(DataSetArchivingStatus.AVAILABLE).getDataSet();
+ lastSeenIdMatcher = prepareListNewerDataSet(ds2);
+ prepareContentProvider(ds2, dataSetFolder1);
+
+ maintenanceTask.execute();
+
+ assertEquals(INFO_PREFIX + "File types: [.txt, .f]\n"
+ + INFO_PREFIX + "BLAST databases folder: " + blastDatabasesFolder + "\n"
+ + INFO_PREFIX + "Temp folder '" + tempFolder + "' created.\n"
+ + INFO_PREFIX + "Scan 1 data sets for creating BLAST databases.\n"
+ + INFO_PREFIX + "Scan 1 data sets for creating BLAST databases.", logRecorder.getLogContent());
+ assertEquals("/usr/bin/blast/makeblastdb -version\n"
+ + "/usr/bin/blast/makeblastdb -in " + tempFolder.getAbsolutePath() + "/DS1-nucl.fa"
+ + " -dbtype nucl -title DS1-nucl -out " + blastDatabasesFolder.getAbsolutePath() + "/DS1-nucl\n"
+ + "/usr/bin/blast/makeblastdb -in " + tempFolder.getAbsolutePath() + "/DS2-nucl.fa"
+ + " -dbtype nucl -title DS2-nucl -out " + blastDatabasesFolder.getAbsolutePath() + "/DS2-nucl\n",
+ maintenanceTask.getCommands());
+ assertEquals("TITLE all-nucl\nDBLIST DS1-nucl DS2-nucl",
+ FileUtilities.loadToString(new File(blastDatabasesFolder, "all-nucl.nal")).trim());
+ assertEquals(false, new File(store, "blast-databases/all-prot.nal").exists());
+ assertEquals("[]", Arrays.asList(tempFolder.listFiles()).toString());
+ assertEquals(11L, lastSeenIdMatcher.recordedObject().getLastSeenDataSetId());
+ assertEquals("12", FileUtilities.loadToString(lastSeenFile).trim());
+ context.assertIsSatisfied();
+ }
+
+ private void prepareContentProvider(final AbstractExternalData dataSet, final File dataSetFolder)
+ {
+ context.checking(new Expectations()
+ {
+ {
+ one(contentProvider).asContent(dataSet);
+ will(returnValue(new DefaultFileBasedHierarchicalContentFactory().asHierarchicalContent(dataSetFolder, null)));
+ }
+ });
+ }
+
+ private RecordingMatcher<TrackingDataSetCriteria> prepareListNewerDataSet(final AbstractExternalData... dataSets)
+ {
+ final RecordingMatcher<TrackingDataSetCriteria> recordingMatcher = new RecordingMatcher<TrackingDataSetCriteria>();
+ context.checking(new Expectations()
+ {
+ {
+ one(service).listNewerDataSets(with(recordingMatcher));
+ will(returnValue(new ArrayList<AbstractExternalData>(Arrays.asList(dataSets))));
+ }
+ });
+ return recordingMatcher;
+ }
+
+}