From 3f7eef61bc7e948464da5d23f1f94e1ed97b0b0e Mon Sep 17 00:00:00 2001
From: tpylak <tpylak>
Date: Wed, 17 Jun 2009 03:01:15 +0000
Subject: [PATCH] LMS-940 yeastx: send emails with error notifications, improve
error messages
SVN: 11401
---
.../PropertiesBasedETLServerPlugin.java | 2 +-
rtd_yeastx/.classpath | 1 +
.../cisd/yeastx/etl/BatchDataSetHandler.java | 86 +++++++++----
.../yeastx/etl/BatchDataSetInfoExtractor.java | 10 +-
.../etl/DataSetMappingInformationParser.java | 14 +--
.../yeastx/etl/DatasetMappingResolver.java | 55 ++++----
.../cisd/yeastx/etl/DatasetMappingUtil.java | 119 +++++++++++-------
.../ch/systemsx/cisd/yeastx/etl/LogUtils.java | 85 ++++++++++---
.../etl/DataSetInformationParserTest.java | 17 +--
9 files changed, 249 insertions(+), 140 deletions(-)
diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/PropertiesBasedETLServerPlugin.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/PropertiesBasedETLServerPlugin.java
index ad2a7dc1266..63a67da96f3 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/PropertiesBasedETLServerPlugin.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/PropertiesBasedETLServerPlugin.java
@@ -155,7 +155,7 @@ public class PropertiesBasedETLServerPlugin extends ETLServerPlugin
} else
{
return create(IDataSetHandler.class, properties, IDataSetHandler.DATASET_HANDLER_KEY,
- true, primaryDataSetHandler, openbisService);
+ false, primaryDataSetHandler, openbisService);
}
}
diff --git a/rtd_yeastx/.classpath b/rtd_yeastx/.classpath
index 86c092fe801..5ac63c6d1c8 100644
--- a/rtd_yeastx/.classpath
+++ b/rtd_yeastx/.classpath
@@ -17,5 +17,6 @@
<classpathentry kind="lib" path="/libraries/log4j/log4j.jar" sourcepath="/libraries/log4j/src.zip"/>
<classpathentry kind="lib" path="/libraries/spring/spring-beans.jar"/>
<classpathentry kind="lib" path="/libraries/cisd-base/cisd-base-test.jar" sourcepath="/libraries/cisd-base/cisd-base-src.zip"/>
+ <classpathentry kind="lib" path="/libraries/mail/mail.jar"/>
<classpathentry kind="output" path="targets/classes"/>
</classpath>
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/BatchDataSetHandler.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/BatchDataSetHandler.java
index 597ab729006..905330f4885 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/BatchDataSetHandler.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/BatchDataSetHandler.java
@@ -28,9 +28,13 @@ import java.util.Set;
import ch.systemsx.cisd.common.collections.TableMap;
import ch.systemsx.cisd.common.filesystem.FileUtilities;
+import ch.systemsx.cisd.common.mail.IMailClient;
+import ch.systemsx.cisd.common.mail.MailClient;
+import ch.systemsx.cisd.common.utilities.ExtendedProperties;
import ch.systemsx.cisd.etlserver.IDataSetHandler;
import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation;
+import ch.systemsx.cisd.yeastx.etl.DatasetMappingUtil.DataSetMappingInformationFile;
/**
* {@link IDataSetHandler} implementation which for each dataset directory reads all the files
@@ -42,46 +46,81 @@ public class BatchDataSetHandler implements IDataSetHandler
{
private final IDataSetHandler delegator;
+ private final IMailClient mailClient;
+
private final DatasetMappingResolver datasetMappingResolver;
- public BatchDataSetHandler(Properties properties, IDataSetHandler delegator,
+ public BatchDataSetHandler(Properties parentProperties, IDataSetHandler delegator,
IEncapsulatedOpenBISService openbisService)
{
this.delegator = delegator;
- this.datasetMappingResolver = new DatasetMappingResolver(properties, openbisService);
+ this.mailClient = new MailClient(parentProperties);
+ this.datasetMappingResolver =
+ new DatasetMappingResolver(getSpecificProperties(parentProperties), openbisService);
+ }
+
+ private static Properties getSpecificProperties(Properties properties)
+ {
+ return ExtendedProperties.getSubset(properties, IDataSetHandler.DATASET_HANDLER_KEY + '.',
+ true);
}
public List<DataSetInformation> handleDataSet(File datasetsParentDir)
{
- List<DataSetInformation> processedDatasetFiles = new ArrayList<DataSetInformation>();
if (canBatchBeProcessed(datasetsParentDir) == false)
{
- return processedDatasetFiles;
+ return createEmptyResult();
}
LogUtils log = new LogUtils(datasetsParentDir);
- TableMap<String, DataSetMappingInformation> datasetsMapping =
- DatasetMappingUtil.tryGetDatasetsMapping(datasetsParentDir);
- if (datasetsMapping == null)
+ DataSetMappingInformationFile datasetMappingFile =
+ DatasetMappingUtil.tryGetDatasetsMapping(datasetsParentDir, log);
+ if (datasetMappingFile == null || datasetMappingFile.tryGetMappings() == null)
{
touchErrorMarkerFile(datasetsParentDir, log);
- return processedDatasetFiles;
+ sendNotificationsIfNecessary(log, tryGetEmail(datasetMappingFile));
+ return createEmptyResult();
}
+ return processDatasets(datasetsParentDir, log, datasetMappingFile.tryGetMappings(),
+ datasetMappingFile.getNotificationEmail());
+ }
+
+ private List<DataSetInformation> processDatasets(File datasetsParentDir, LogUtils log,
+ TableMap<String, DataSetMappingInformation> mappings, String notificationEmail)
+ {
+ List<DataSetInformation> processedDatasetFiles = createEmptyResult();
+
Set<String> processedFiles = new HashSet<String>();
List<File> files = listAll(datasetsParentDir);
for (File file : files)
{
- if (canDatasetBeProcessed(file, datasetsMapping, log))
+ if (canDatasetBeProcessed(file, mappings, log))
{
processedDatasetFiles.addAll(delegator.handleDataSet(file));
processedFiles.add(file.getName().toLowerCase());
}
}
- clean(datasetsParentDir, processedFiles, log, datasetsMapping.values().size());
- log.sendNotificationsIfNecessary();
+ clean(datasetsParentDir, processedFiles, log, mappings.values().size());
+ sendNotificationsIfNecessary(log, notificationEmail);
return processedDatasetFiles;
}
- private boolean canBatchBeProcessed(File parentDir)
+ private void sendNotificationsIfNecessary(LogUtils log, String email)
+ {
+ log.sendNotificationsIfNecessary(mailClient, email);
+ }
+
+ private static String tryGetEmail(DataSetMappingInformationFile datasetMappingFileOrNull)
+ {
+ return datasetMappingFileOrNull == null ? null : datasetMappingFileOrNull
+ .getNotificationEmail();
+ }
+
+ private static ArrayList<DataSetInformation> createEmptyResult()
+ {
+ return new ArrayList<DataSetInformation>();
+ }
+
+ private static boolean canBatchBeProcessed(File parentDir)
{
if (parentDir.isDirectory() == false)
{
@@ -109,18 +148,13 @@ public class BatchDataSetHandler implements IDataSetHandler
return new File(datasetsParentDir, ERROR_MARKER_FILE).isFile();
}
- private void cleanMappingFile(File datasetsParentDir, Set<String> processedFiles, LogUtils log)
+ private static void cleanMappingFile(File datasetsParentDir, Set<String> processedFiles,
+ LogUtils log)
{
- try
- {
- DatasetMappingUtil.cleanMappingFile(datasetsParentDir, processedFiles);
- } catch (IOException ex)
- {
- log.userError("Cannot clean dataset mappings file: " + ex.getMessage());
- }
+ DatasetMappingUtil.cleanMappingFile(datasetsParentDir, processedFiles, log);
}
- private void clean(File datasetsParentDir, Set<String> processedFiles, LogUtils log,
+ private static void clean(File datasetsParentDir, Set<String> processedFiles, LogUtils log,
int datasetMappingsNumber)
{
cleanMappingFile(datasetsParentDir, processedFiles, log);
@@ -155,13 +189,13 @@ public class BatchDataSetHandler implements IDataSetHandler
.getPath());
} else
{
- log.userWarning(
+ log.warning(
"Correct the errors and delete the '%s' file to start processing again.",
ERROR_MARKER_FILE);
}
}
- private void cleanDatasetsDir(File datasetsParentDir)
+ private static void cleanDatasetsDir(File datasetsParentDir)
{
LogUtils.deleteUserLog(datasetsParentDir);
DatasetMappingUtil.deleteMappingFile(datasetsParentDir);
@@ -186,7 +220,7 @@ public class BatchDataSetHandler implements IDataSetHandler
return datasetMappingResolver.isMappingCorrect(mapping, log);
}
- private void deleteEmptyDir(File dir)
+ private static void deleteEmptyDir(File dir)
{
boolean ok = dir.delete();
if (ok == false)
@@ -197,7 +231,7 @@ public class BatchDataSetHandler implements IDataSetHandler
}
}
- private boolean hasNoPotentialDatasetFiles(File dir)
+ private static boolean hasNoPotentialDatasetFiles(File dir)
{
List<File> files = listAll(dir);
int datasetsCounter = files.size();
@@ -211,7 +245,7 @@ public class BatchDataSetHandler implements IDataSetHandler
return datasetsCounter == 0;
}
- private List<File> listAll(File dataSet)
+ private static List<File> listAll(File dataSet)
{
return FileUtilities.listFilesAndDirectories(dataSet, false, null);
}
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/BatchDataSetInfoExtractor.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/BatchDataSetInfoExtractor.java
index 8a1a974929d..62779d1c2ad 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/BatchDataSetInfoExtractor.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/BatchDataSetInfoExtractor.java
@@ -42,15 +42,15 @@ public class BatchDataSetInfoExtractor implements IDataSetInfoExtractor
IEncapsulatedOpenBISService openbisService) throws UserFailureException,
EnvironmentFailureException
{
+ LogUtils log = new LogUtils(incomingDataSetPath.getParentFile());
DataSetMappingInformation plainInfo =
- DatasetMappingUtil.tryGetDatasetMapping(incomingDataSetPath);
+ DatasetMappingUtil.tryGetDatasetMapping(incomingDataSetPath, log);
if (plainInfo != null)
{
DataSetInformationYeastX info = new DataSetInformationYeastX();
info.setComplete(true);
info.setDataSetProperties(plainInfo.getProperties());
- String sampleCode =
- getSampleCode(plainInfo, openbisService, incomingDataSetPath.getParentFile());
+ String sampleCode = getSampleCode(plainInfo, openbisService, log);
info.setSampleCode(sampleCode);
info.setGroupCode(plainInfo.getGroupCode());
MLConversionType conversion = getConversion(plainInfo.getConversion());
@@ -76,11 +76,11 @@ public class BatchDataSetInfoExtractor implements IDataSetInfoExtractor
}
private String getSampleCode(DataSetMappingInformation mapping,
- IEncapsulatedOpenBISService openbisService, File logDir)
+ IEncapsulatedOpenBISService openbisService, LogUtils log)
{
String sampleCode =
new DatasetMappingResolver(properties, openbisService).tryFigureSampleCode(mapping,
- new LogUtils(logDir));
+ log);
if (sampleCode == null)
{
// should not happen, the dataset handler should skip datasets with incorrect mapping
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DataSetMappingInformationParser.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DataSetMappingInformationParser.java
index f87a74cb737..03003ec3d74 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DataSetMappingInformationParser.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DataSetMappingInformationParser.java
@@ -36,7 +36,7 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.NewProperty;
*/
class DataSetMappingInformationParser
{
- public static List<DataSetMappingInformation> tryParse(File mappingFile)
+ public static List<DataSetMappingInformation> tryParse(File mappingFile, LogUtils log)
{
TabFileLoader<DataSetMappingInformation> tabFileLoader =
new TabFileLoader<DataSetMappingInformation>(
@@ -54,16 +54,16 @@ class DataSetMappingInformationParser
return tabFileLoader.load(mappingFile);
} catch (final IllegalArgumentException e)
{
- logParsingError(e, mappingFile);
+ logParsingError(log, e, mappingFile);
return null;
} catch (final Exception e)
{
- logParsingError(e, mappingFile);
+ logParsingError(log, e, mappingFile);
return null;
}
}
- private static void logParsingError(Exception e, File mappingFile)
+ private static void logParsingError(LogUtils log, Exception e, File mappingFile)
{
Throwable cause = e.getCause();
String causeMsg = "";
@@ -71,10 +71,8 @@ class DataSetMappingInformationParser
{
causeMsg = "\nThe exception was caused by: " + cause.getMessage();
}
- LogUtils.basicError(mappingFile.getParentFile(),
- "The datasets cannot be processed because the mapping file '%s' has incorrect format."
- + " The following exception occured:\n%s%s", mappingFile.getPath(), e
- .getMessage(), causeMsg);
+ log.mappingFileError(mappingFile, "the mapping file '%s' has incorrect format."
+ + " The problem is:\n%s%s", mappingFile.getPath(), e.getMessage(), causeMsg);
}
private static final class NewPropertyParserObjectFactory extends
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DatasetMappingResolver.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DatasetMappingResolver.java
index be8d4350427..7002768232e 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DatasetMappingResolver.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DatasetMappingResolver.java
@@ -101,7 +101,7 @@ class DatasetMappingResolver
samples = openbisService.listSamplesByCriteria(criteria);
} catch (UserFailureException e)
{
- logMappingError(mapping, log, e.getMessage());
+ log.datasetMappingError(mapping, e.getMessage());
return null;
}
if (samples.size() == 1)
@@ -109,7 +109,7 @@ class DatasetMappingResolver
return samples.get(0);
} else if (samples.size() == 0)
{
- logMappingError(mapping, log, "there is no sample which matches the criteria <"
+ log.datasetMappingError(mapping, "there is no sample which matches the criteria <"
+ criteria + ">");
return null;
} else
@@ -119,7 +119,7 @@ class DatasetMappingResolver
"there should be exacty one sample which matches the criteria '%s', but %d of them were found."
+ " Consider using the unique sample code.", criteria, samples
.size());
- logMappingError(mapping, log, errMsg);
+ log.datasetMappingError(mapping, errMsg);
return null;
}
}
@@ -160,25 +160,26 @@ class DatasetMappingResolver
MLConversionType conversion = MLConversionType.tryCreate(conversionText);
if (conversion == null)
{
- log.userError(String.format(
- "Error for file '%s'. Unexpected value '%s' in 'conversion' column. "
- + "Leave the column empty or use one of the allowed values: %s.",
- mapping.getFileName(), conversionText, CollectionUtils.abbreviate(
- MLConversionType.values(), MLConversionType.values().length)));
+ String availableConvTypes =
+ CollectionUtils.abbreviate(MLConversionType.values(),
+ MLConversionType.values().length);
+ log.datasetMappingError(mapping, "unexpected value '%s' in 'conversion' column. "
+ + "Leave the column empty or use one of the allowed values: %s.",
+ conversionText, availableConvTypes);
return false;
}
boolean conversionRequired = isConversionRequired(mapping);
if (conversion == MLConversionType.NONE && conversionRequired)
{
- log.userError("Error for file '%s'. Conversion column cannot be empty "
- + "for this type of file.", mapping.getFileName());
+ log.datasetMappingError(mapping, "conversion column cannot be empty "
+ + "for this type of file.");
return false;
}
if (conversion != MLConversionType.NONE && conversionRequired == false)
{
- log.userError("Error for file '%s'. Conversion column must be empty "
- + "for this type of file.", mapping.getFileName());
+ log.datasetMappingError(mapping, "conversion column must be empty "
+ + "for this type of file.");
return false;
}
return true;
@@ -197,8 +198,8 @@ class DatasetMappingResolver
ExperimentPE experiment = tryFigureExperiment(sampleCode, mapping, log);
if (experiment == null)
{
- logMappingError(mapping, log, String.format("sample with the code '%s' does not exist"
- + " or is not connected to any experiment", sampleCode));
+ log.datasetMappingError(mapping, "sample with the code '%s' does not exist"
+ + " or is not connected to any experiment", sampleCode);
return false;
}
return true;
@@ -213,7 +214,8 @@ class DatasetMappingResolver
return openbisService.getBaseExperiment(sampleIdentifier);
} catch (UserFailureException e)
{
- log.userError("Error when checking if sample '%s' belongs to an experiment: %s",
+ log.datasetMappingError(mapping,
+ "error when checking if sample '%s' belongs to an experiment: %s",
sampleIdentifier, e.getMessage());
return null;
}
@@ -230,29 +232,24 @@ class DatasetMappingResolver
{
if ((mapping.getExperimentCode() == null) != (mapping.getProjectCode() == null))
{
- logMappingError(mapping, log,
- "experiment and project columns should be both empty or should be both filled.");
+ log
+ .datasetMappingError(mapping,
+ "experiment and project columns should be both empty or should be both filled.");
return false;
}
if (propertyCodeOrNull == null && mapping.getExperimentCode() != null)
{
- logMappingError(
- mapping,
- log,
- "openBis is not configured to use the sample label to identify the sample."
- + " You can still identify the sample by the code (clear the experiment column in this case)."
- + " You can also contact your administrator to change the server configuration and set the property type code which should be used.");
+ log
+ .datasetMappingError(
+ mapping,
+ "openBis is not configured to use the sample label to identify the sample."
+ + " You can still identify the sample by the code (clear the experiment column in this case)."
+ + " You can also contact your administrator to change the server configuration and set the property type code which should be used.");
return false;
}
return true;
}
- private void logMappingError(DataSetMappingInformation mapping, LogUtils log,
- String errorMessage)
- {
- log.userError("Mapping for file " + mapping.getFileName() + " is incorrect: " + errorMessage);
- }
-
public static void adaptPropertyCodes(List<DataSetMappingInformation> list)
{
for (DataSetMappingInformation mapping : list)
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DatasetMappingUtil.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DatasetMappingUtil.java
index 895b1eeda37..ad0e69e1332 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DatasetMappingUtil.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/DatasetMappingUtil.java
@@ -48,7 +48,7 @@ class DatasetMappingUtil
{ "tsv" };
private static TableMap<String, DataSetMappingInformation> tryAsFileMap(
- List<DataSetMappingInformation> list, File logDir)
+ List<DataSetMappingInformation> list, LogUtils log, File mappingFile)
{
IKeyExtractor<String, DataSetMappingInformation> extractor =
new IKeyExtractor<String, DataSetMappingInformation>()
@@ -64,23 +64,21 @@ class DatasetMappingUtil
UniqueKeyViolationStrategy.ERROR);
} catch (UniqueKeyViolationException e)
{
- // TODO 2009-06-16, Tomasz Pylak: use email to send notifications
- LogUtils.basicError(logDir,
- "The file '%s' appears more than once. No datasets will be processed.", e
- .getInvalidKey());
+ log.mappingFileError(mappingFile, "the file '%s' appears more than once.", e
+ .getInvalidKey());
return null;
}
}
- public static DataSetMappingInformation tryGetDatasetMapping(File datasetFile)
+ public static DataSetMappingInformation tryGetDatasetMapping(File datasetFile, LogUtils log)
{
- TableMap<String, DataSetMappingInformation> datasetsMapping =
- tryGetDatasetsMapping(datasetFile.getParentFile());
- if (datasetsMapping == null)
+ DataSetMappingInformationFile datasetsMappings =
+ tryGetDatasetsMapping(datasetFile.getParentFile(), log);
+ if (datasetsMappings == null || datasetsMappings.tryGetMappings() == null)
{
return null;
}
- return tryGetDatasetMapping(datasetFile, datasetsMapping);
+ return tryGetDatasetMapping(datasetFile, datasetsMappings.tryGetMappings());
}
public static DataSetMappingInformation tryGetDatasetMapping(File datasetFile,
@@ -91,7 +89,7 @@ class DatasetMappingUtil
}
// returns the content of the first line comment or null if there is no comment or it is empty
- private static String tryGetFirstLineCommentContent(File mappingFile)
+ private static String tryGetFirstLineCommentContent(File mappingFile, LogUtils log)
{
List<String> lines;
try
@@ -99,7 +97,8 @@ class DatasetMappingUtil
lines = readLines(mappingFile);
} catch (IOException e)
{
- errorInFile(mappingFile, e.getMessage());
+ Object[] arguments = {};
+ log.mappingFileError(mappingFile, e.getMessage(), arguments);
return null;
}
if (lines.size() == 0)
@@ -120,60 +119,80 @@ class DatasetMappingUtil
return firstLine;
}
- private static void errorInFile(File file, String message)
- {
- LogUtils.basicError(file.getParentFile(), "Error while reading the file '%s': %s", file
- .getPath(), message);
- }
-
- // returns email address from the first line of the mapping file or null if there is no emai or
- // it is invalid
- private static String tryGetEmail(File mappingFile)
+ /**
+ * @return email address from the first line of the mapping file or null if there is no emai or
+ * it is invalid.
+ */
+ private static String tryGetEmail(File mappingFile, LogUtils log)
{
- String email = tryGetFirstLineCommentContent(mappingFile);
+ String email = tryGetFirstLineCommentContent(mappingFile, log);
if (email == null)
{
- errorInFile(mappingFile, String.format(
+ log.mappingFileError(mappingFile,
"There should be a '%s' character followed by an email address "
+ "in the first line of the file. "
+ "The email is needed to send messages about errors.",
- TabFileLoader.COMMENT_PREFIX));
+ TabFileLoader.COMMENT_PREFIX);
return null;
}
if (email.contains("@") == false || email.contains(".") == false)
{
- errorInFile(mappingFile, String.format(
- "The text '%s' does not seem to be an email address.", email));
+ log.mappingFileError(mappingFile,
+ "The text '%s' does not seem to be an email address.", email);
return null;
}
return email;
}
- public static TableMap<String/* file name in lowercase */, DataSetMappingInformation> tryGetDatasetsMapping(
- File parentDir)
+ static class DataSetMappingInformationFile
+ {
+ private final TableMap<String/* file name in lowercase */, DataSetMappingInformation> mappingsOrNull;
+
+ private final String notificationEmail;
+
+ public DataSetMappingInformationFile(
+ TableMap<String, DataSetMappingInformation> mappingsOrNull, String notificationEmail)
+ {
+ this.mappingsOrNull = mappingsOrNull;
+ this.notificationEmail = notificationEmail;
+ }
+
+ public TableMap<String, DataSetMappingInformation> tryGetMappings()
+ {
+ return mappingsOrNull;
+ }
+
+ public String getNotificationEmail()
+ {
+ return notificationEmail;
+ }
+ }
+
+ public static DataSetMappingInformationFile tryGetDatasetsMapping(File parentDir, LogUtils log)
{
File mappingFile = tryGetMappingFile(parentDir);
if (mappingFile == null)
{
- LogUtils.basicWarn(parentDir, "Cannot process the directory '%s' "
+ log.error("No datasets from the directory '%s' can be processed "
+ "because a file with extension '%s' which contains dataset descriptions "
- + "does not exist or there is more than one.", parentDir.getPath(),
- CollectionUtils.abbreviate(MAPPING_FILE_EXTENSIONS, -1));
+ + "does not exist or there is more than one file with taht extension.",
+ parentDir.getName(), CollectionUtils.abbreviate(MAPPING_FILE_EXTENSIONS, -1));
return null;
}
- String notificationEmail = tryGetEmail(mappingFile);
+ String notificationEmail = tryGetEmail(mappingFile, log);
if (notificationEmail == null)
{
- return null;
+ return null; // email has to be provided always
}
+ TableMap<String, DataSetMappingInformation> mappingsOrNull = null;
List<DataSetMappingInformation> list =
- DataSetMappingInformationParser.tryParse(mappingFile);
- if (list == null)
+ DataSetMappingInformationParser.tryParse(mappingFile, log);
+ if (list != null)
{
- return null;
+ DatasetMappingResolver.adaptPropertyCodes(list);
+ mappingsOrNull = tryAsFileMap(list, log, mappingFile);
}
- DatasetMappingResolver.adaptPropertyCodes(list);
- return tryAsFileMap(list, parentDir);
+ return new DataSetMappingInformationFile(mappingsOrNull, notificationEmail);
}
public static boolean isMappingFile(File file)
@@ -228,25 +247,31 @@ class DatasetMappingUtil
*
* @param processedFiles files which should be removed from the mapping file
*/
- public static void cleanMappingFile(File parentDir, Set<String> processedFiles)
- throws IOException
+ public static void cleanMappingFile(File parentDir, Set<String> processedFiles, LogUtils log)
{
File mappingFile = tryGetMappingFile(parentDir);
if (mappingFile == null)
{
return;
}
- List<String> lines = readLines(mappingFile);
- List<String> unprocessedLines = new ArrayList<String>();
- for (String line : lines)
+ try
{
- String tokens[] = line.trim().split("\t");
- if (tokens.length == 0 || processedFiles.contains(tokens[0].toLowerCase()) == false)
+ List<String> lines = readLines(mappingFile);
+ List<String> unprocessedLines = new ArrayList<String>();
+ for (String line : lines)
{
- unprocessedLines.add(line);
+ String tokens[] = line.trim().split("\t");
+ if (tokens.length == 0 || processedFiles.contains(tokens[0].toLowerCase()) == false)
+ {
+ unprocessedLines.add(line);
+ }
}
+ IOUtils.writeLines(unprocessedLines, "\n", new FileOutputStream(mappingFile));
+ } catch (IOException ex)
+ {
+ log.mappingFileError(mappingFile, "cannot clean dataset mappings file: "
+ + ex.getMessage());
}
- IOUtils.writeLines(unprocessedLines, "\n", new FileOutputStream(mappingFile));
}
@SuppressWarnings("unchecked")
diff --git a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/LogUtils.java b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/LogUtils.java
index c7664344e46..5dfe772b085 100644
--- a/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/LogUtils.java
+++ b/rtd_yeastx/source/java/ch/systemsx/cisd/yeastx/etl/LogUtils.java
@@ -28,12 +28,16 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.mail.IMailClient;
/**
* @author Tomasz Pylak
*/
class LogUtils
{
+ private static final String ERROR_NOTIFICATION_EMAIL_SUBJECT =
+ "[openBIS] problems with datasets upload in directory: %s";
+
private static final Logger notificationLog =
LogFactory.getLogger(LogCategory.NOTIFY, LogUtils.class);
@@ -42,50 +46,97 @@ class LogUtils
private final File loggingDir;
- private final StringBuffer messageToSend;
+ private final StringBuffer errorMessages;
public LogUtils(File loggingDir)
{
this.loggingDir = loggingDir;
- this.messageToSend = new StringBuffer();
+ this.errorMessages = new StringBuffer();
+ }
+
+ /** Logs errors about one dataset mapping. Uses user log file and email notification. */
+ public void datasetMappingError(DataSetMappingInformation mapping, String errorMessageFormat,
+ Object... arguments)
+ {
+ String errorMessage = String.format(errorMessageFormat, arguments);
+ error("Cannot upload the file " + mapping.getFileName() + ": " + errorMessage);
}
- public void userError(String messageFormat, Object... arguments)
+ /**
+ * Logs an error about the syntax of the mapping file. Uses user log file and email
+ * notification.
+ */
+ public void mappingFileError(File mappingFile, String messageFormat, Object... arguments)
{
- String message = basicError(loggingDir, messageFormat, arguments);
- messageToSend.append(message);
+ String errorMessage = String.format(messageFormat, arguments);
+ error("No datasets could be processed, because there is an error in the mapping file "
+ + mappingFile.getName() + ": " + errorMessage);
}
- public void userWarning(String messageFormat, Object... arguments)
+ /** Uses user log file and email notification to log an error. */
+ public void error(String messageFormat, Object... arguments)
{
- String message = basicWarn(loggingDir, messageFormat, arguments);
- messageToSend.append(message);
+ logError(loggingDir, messageFormat, arguments);
+ appendNotification(messageFormat, arguments);
}
- public void sendNotificationsIfNecessary()
+ /** Uses user log file and email notification to log a warning. */
+ public void warning(String messageFormat, Object... arguments)
{
- // TODO 2009-06-16, Tomasz Pylak: add email notification
- if (messageToSend.length() > 0)
+ logWarning(loggingDir, messageFormat, arguments);
+ appendNotification(messageFormat, arguments);
+ }
+
+ private void appendNotification(String messageFormat, Object... arguments)
+ {
+ errorMessages.append(String.format(messageFormat, arguments));
+ errorMessages.append("\n");
+ }
+
+ /** has to be called at the end to send all notifications in one email */
+ public void sendNotificationsIfNecessary(IMailClient mailClient, String notificationEmailOrNull)
+ {
+ if (notificationEmailOrNull != null && errorMessages.length() > 0)
{
- System.out.println("Email content: ");
- System.out.println(messageToSend);
+ sendErrorMessage(mailClient, notificationEmailOrNull);
}
}
+ private void sendErrorMessage(IMailClient mailClient, String notificationEmail)
+ {
+ String subject = String.format(ERROR_NOTIFICATION_EMAIL_SUBJECT, loggingDir.getName());
+ mailClient.sendMessage(subject, createErrorNotificationContent(), null, notificationEmail);
+ }
+
+ private String createErrorNotificationContent()
+ {
+ StringBuffer sb = new StringBuffer();
+ sb.append("Hello,\n");
+ sb.append("This email has been generated automatically by openBIS.\n");
+ sb.append("The upload of some datasets from '");
+ sb.append(loggingDir.getName());
+ sb.append("' directory has failed. There are following errors:\n");
+ sb.append(errorMessages);
+ sb.append("\n");
+ sb.append("If you are not sure how to correct the errors and you cannot find the answer"
+ + " in the documentation, ask for help your openBIS administrator.\n");
+ sb.append("Kind regards,\n");
+ sb.append(" openBIS Team");
+ return sb.toString();
+ }
+
/** Adds an entry about an error to the user log file. Does not send an email. */
- public static String basicError(File loggingDir, String messageFormat, Object... arguments)
+ private static void logError(File loggingDir, String messageFormat, Object... arguments)
{
String message = createUserMessage("ERROR", messageFormat, arguments);
notifyUserByLogFile(loggingDir, message);
- return message;
}
/** Adds an entry about a warning to the user log file. Does not send an email. */
- public static String basicWarn(File loggingDir, String messageFormat, Object... arguments)
+ private static void logWarning(File loggingDir, String messageFormat, Object... arguments)
{
String message = createUserMessage("WARNING", messageFormat, arguments);
notifyUserByLogFile(loggingDir, message);
- return message;
}
private static void notifyUserByLogFile(File loggingDir, String message)
diff --git a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/etl/DataSetInformationParserTest.java b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/etl/DataSetInformationParserTest.java
index da2419b10bb..443f1ad36a2 100644
--- a/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/etl/DataSetInformationParserTest.java
+++ b/rtd_yeastx/sourceTest/java/ch/systemsx/cisd/yeastx/etl/DataSetInformationParserTest.java
@@ -47,7 +47,7 @@ public class DataSetInformationParserTest extends AbstractFileSystemTestCase
{
File indexFile =
writeMappingFile(HEADER + "data.txt sample1 group1 experiment1 fiaML v1 v2");
- List<DataSetMappingInformation> list = DataSetMappingInformationParser.tryParse(indexFile);
+ List<DataSetMappingInformation> list = tryParse(indexFile);
AssertJUnit.assertEquals(1, list.size());
DataSetMappingInformation elem = list.get(0);
AssertJUnit.assertEquals("group1", elem.getGroupCode());
@@ -63,14 +63,12 @@ public class DataSetInformationParserTest extends AbstractFileSystemTestCase
IOException
{
File indexFile = writeMappingFile(HEADER + TAB + TAB + TAB + TAB + TAB + TAB);
- List<DataSetMappingInformation> result =
- DataSetMappingInformationParser.tryParse(indexFile);
+ List<DataSetMappingInformation> result = tryParse(indexFile);
AssertJUnit.assertNull("error during parsing expected", result);
List<String> logLines = readLogFile();
System.out.println(logLines);
AssertJUnit.assertEquals(3, logLines.size());
- AssertionUtil.assertContains("Missing value for the mandatory column", logLines
- .get(2));
+ AssertionUtil.assertContains("Missing value for the mandatory column", logLines.get(2));
}
@Test
@@ -78,8 +76,7 @@ public class DataSetInformationParserTest extends AbstractFileSystemTestCase
IOException
{
File indexFile = writeMappingFile("xxx");
- List<DataSetMappingInformation> result =
- DataSetMappingInformationParser.tryParse(indexFile);
+ List<DataSetMappingInformation> result = tryParse(indexFile);
AssertJUnit.assertNull("error during parsing expected", result);
List<String> logLines = readLogFile();
AssertJUnit.assertEquals(2, logLines.size());
@@ -87,6 +84,12 @@ public class DataSetInformationParserTest extends AbstractFileSystemTestCase
"Mandatory column(s) 'group', 'sample', 'file_name' are missing", logLines.get(1));
}
+ private List<DataSetMappingInformation> tryParse(File indexFile)
+ {
+ LogUtils log = new LogUtils(indexFile.getParentFile());
+ return DataSetMappingInformationParser.tryParse(indexFile, log);
+ }
+
private List<String> readLogFile() throws IOException, FileNotFoundException
{
File log = new File(workingDirectory, ConstantsYeastX.USER_LOG_FILE);
--
GitLab