Skip to content
Snippets Groups Projects
Commit 41f0d020 authored by cramakri's avatar cramakri
Browse files

LMS-1947 Better error messages. 

SVN: 19214
parent 225d1b12
No related branches found
No related tags found
No related merge requests found
Expand all Hide whitespace changes
Inline Side-by-side
Showing
with 437 additions and 99 deletions
datastore_server/source/java/ch/systemsx/cisd/etlserver/entityregistration/RegistrationStatus.java 0 → 100644
+ 76
− 0
View file @ 41f0d020
package ch.systemsx.cisd.etlserver.entityregistration;
/**
* An interface to describe success and errors encountered during registration.
*
* @author Chandrasekhar Ramakrishnan
*/
interface IRegistrationStatus
{
boolean isError();
Throwable getError();
String getMessage();
}
/**
* Abstract superclass for success and errors.
*
* @author Chandrasekhar Ramakrishnan
*/
abstract class AbstractRegistrationStatus implements IRegistrationStatus
{
protected AbstractRegistrationStatus()
{
}
}
class RegistrationError extends AbstractRegistrationStatus
{
private final Throwable error;
RegistrationError(Throwable error)
{
this.error = error;
}
public boolean isError()
{
return true;
}
public Throwable getError()
{
return error;
}
public String getMessage()
{
return error.getMessage();
}
}
class RegistrationSuccess extends AbstractRegistrationStatus
{
RegistrationSuccess(String[] registeredMetadata)
{
}
public boolean isError()
{
return false;
}
public Throwable getError()
{
assert false : "getError() should not be called on a success object";
return null;
}
public String getMessage()
{
return "Success";
}
}
datastore_server/source/java/ch/systemsx/cisd/etlserver/entityregistration/SampleAndDataSetControlFileProcessor.java
+ 192
− 19
View file @ 41f0d020
......@@ -21,7 +21,9 @@ import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.Reader;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import javax.activation.DataHandler;
......@@ -34,7 +36,6 @@ import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.io.DelegatedReader;
import ch.systemsx.cisd.common.mail.EMailAddress;
import ch.systemsx.cisd.common.utilities.UnicodeUtils;
import ch.systemsx.cisd.etlserver.entityregistration.SampleAndDataSetRegistrator.RegistrationErrorWrapper;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DataSetType;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.Person;
import ch.systemsx.cisd.openbis.generic.shared.basic.dto.SampleType;
......@@ -47,10 +48,26 @@ import ch.systemsx.cisd.openbis.generic.shared.parser.GlobalPropertiesLoader;
*/
class SampleAndDataSetControlFileProcessor extends AbstractSampleAndDataSetProcessor
{
private static final String SUCCESS_FILENAME = "registered.txt";
private final File controlFile;
private final HashMap<SampleDataSetPair, RegistrationErrorWrapper> errorMap =
new HashMap<SampleDataSetPair, RegistrationErrorWrapper>();
private final HashMap<SampleDataSetPair, IRegistrationStatus> errorMap =
new HashMap<SampleDataSetPair, IRegistrationStatus>();
private final HashMap<SampleDataSetPair, IRegistrationStatus> successMap =
new HashMap<SampleDataSetPair, IRegistrationStatus>();
private final HashSet<File> processedDataSetFiles = new HashSet<File>();
// State that is filled out as a result of processing
private ControlFileRegistrationProperties properties;
private String failureLinesResultSectionOrNull;
private String unmentionedSubfoldersResultSectionOrNull;
private String successLinesResultSectionOrNull;
/**
* Utility class for accessing the properties defined in a control file.
......@@ -226,8 +243,7 @@ class SampleAndDataSetControlFileProcessor extends AbstractSampleAndDataSetProce
logControlFileOverridePropertiesExtracted(overrideProperties);
ControlFileRegistrationProperties properties =
new ControlFileRegistrationProperties(overrideProperties, globalState);
properties = new ControlFileRegistrationProperties(overrideProperties, globalState);
BisTabFileLoader<SampleDataSetPair> controlFileLoader =
new BisTabFileLoader<SampleDataSetPair>(
......@@ -252,7 +268,7 @@ class SampleAndDataSetControlFileProcessor extends AbstractSampleAndDataSetProce
throw e;
}
sendEmailWithErrorMessage(properties, e.getMessage());
sendEmailWithErrorMessage(e.getMessage());
return;
} finally
{
......@@ -272,26 +288,69 @@ class SampleAndDataSetControlFileProcessor extends AbstractSampleAndDataSetProce
sampleDataSet.getDataSetInformation().setUploadingUserId(userId);
SampleAndDataSetRegistrator registrator =
new SampleAndDataSetRegistrator(folder, properties, sampleDataSet);
RegistrationErrorWrapper resultOrNull = registrator.register();
if (null != resultOrNull)
IRegistrationStatus result = registrator.register();
processedDataSetFiles.add(registrator.getDataSetFile());
if (result.isError())
{
errorMap.put(sampleDataSet, resultOrNull);
errorMap.put(sampleDataSet, result);
} else
{
successMap.put(sampleDataSet, result);
}
}
if (false == errorMap.isEmpty())
sendResultsEmail();
}
private void sendResultsEmail()
{
createFailureLinesSection();
createUnmentionedFoldersSection();
createSuccessfulLinesSection();
boolean wasSuccessful = true;
StringBuilder resultEmail = new StringBuilder();
if (null != failureLinesResultSectionOrNull)
{
resultEmail.append(failureLinesResultSectionOrNull);
resultEmail.append("\n");
wasSuccessful = false;
}
if (null != unmentionedSubfoldersResultSectionOrNull)
{
resultEmail.append(unmentionedSubfoldersResultSectionOrNull);
resultEmail.append("\n");
wasSuccessful = false;
}
if (null != successLinesResultSectionOrNull)
{
sendEmailWithErrorMessage(properties, createErrorMessageFromErrorMap());
resultEmail.append(successLinesResultSectionOrNull);
}
if (wasSuccessful)
{
sendEmailWithSuccessMessage(resultEmail.toString());
} else
{
sendEmailWithErrorMessage(resultEmail.toString());
}
}
private String createErrorMessageFromErrorMap()
private void createFailureLinesSection()
{
if (errorMap.isEmpty())
{
failureLinesResultSectionOrNull = null;
return;
}
StringBuilder sb = new StringBuilder();
sb.append("Encountered errors in the following lines:\n");
for (SampleDataSetPair pair : errorMap.keySet())
{
RegistrationErrorWrapper error = errorMap.get(pair);
IRegistrationStatus error = errorMap.get(pair);
sb.append("# ");
sb.append(error.getMessage());
sb.append("\n");
......@@ -308,26 +367,107 @@ class SampleAndDataSetControlFileProcessor extends AbstractSampleAndDataSetProce
}
sb.append("\n");
}
return sb.toString();
failureLinesResultSectionOrNull = sb.toString();
}
private void createUnmentionedFoldersSection()
{
if (false == globalState.areUnmentionedFoldersAnError())
{
unmentionedSubfoldersResultSectionOrNull = null;
return;
}
// Make sure all folders were processed
ArrayList<File> unprocessedFiles = getUnprocessedDataSetList();
if (unprocessedFiles.isEmpty())
{
unmentionedSubfoldersResultSectionOrNull = null;
return;
}
StringBuilder sb = new StringBuilder();
sb.append("The following subfolders were in the uploaded folder, but were not mentioned in the control file:\n");
for (File file : unprocessedFiles)
{
sb.append(file.getName());
sb.append(",");
}
// remove the final comma
sb.deleteCharAt(sb.length() - 1);
unmentionedSubfoldersResultSectionOrNull = sb.toString();
}
private void createSuccessfulLinesSection()
{
if (errorMap.isEmpty())
{
successLinesResultSectionOrNull = null;
return;
}
StringBuilder sb = new StringBuilder();
sb.append("The following lines were successfully registered:\n");
for (SampleDataSetPair pair : successMap.keySet())
{
sb.append("# ");
String[] tokens = pair.getTokens();
int i = 0;
for (String token : tokens)
{
sb.append(token);
if (++i < tokens.length)
{
sb.append("\t");
}
}
sb.append("\n");
}
successLinesResultSectionOrNull = sb.toString();
}
private void logControlFileOverridePropertiesExtracted(ControlFileOverrideProperties properties)
private void logControlFileOverridePropertiesExtracted(
ControlFileOverrideProperties overrideProperties)
{
String message =
String.format(
"Global properties extracted from file '%s': SAMPLE_TYPE(%s) DATA_SET_TYPE(%s) USER(%s)",
controlFile.getName(), properties.trySampleType(),
properties.tryDataSetType(), properties.tryUserString());
controlFile.getName(), overrideProperties.trySampleType(),
overrideProperties.tryDataSetType(), overrideProperties.tryUserString());
logInfo(message);
}
/**
* Send an email message to the person who uploaded the file, telling them that everything went
* ok.
*/
private void sendEmailWithSuccessMessage(String message)
{
// Create an email and send it.
try
{
String subject = createSuccessEmailSubject();
String content = createSuccessEmailContent();
String filename = SUCCESS_FILENAME;
EMailAddress recipient = new EMailAddress(properties.getUser().getEmail());
DataSource dataSource = new ByteArrayDataSource(message, "text/plain");
globalState.getMailClient().sendEmailMessageWithAttachment(subject, content, filename,
new DataHandler(dataSource), null, null, recipient);
} catch (IOException ex)
{
throw CheckedExceptionTunnel.wrapIfNecessary(ex);
}
}
/**
* Send an email message to the person who uploaded the file. This method is only called if we
* have a valid email address to contact. Otherwise, errors are forwarded to a higher level for
* handling.
*/
private void sendEmailWithErrorMessage(ControlFileRegistrationProperties properties,
String message)
private void sendEmailWithErrorMessage(String message)
{
// Log it
logError(message);
......@@ -360,4 +500,37 @@ class SampleAndDataSetControlFileProcessor extends AbstractSampleAndDataSetProce
.format("Not all samples and data sets specified in the control file, %s, could be registered / updated. The errors are detailed in the attachment. Each faulty line is reproduced, preceded by a comment explaining the cause of the error.",
controlFile);
}
private String createSuccessEmailSubject()
{
return String.format("Sample / Data Set Registration Succeeded -- %s", controlFile);
}
private String createSuccessEmailContent()
{
return String
.format("The registration/update of samples and the registration of data sets was successful specified in the control file, %s, was successful.",
controlFile);
}
private ArrayList<File> getUnprocessedDataSetList()
{
File[] files = folder.listFiles();
ArrayList<File> unprcessedDataSets = new ArrayList<File>();
for (File file : files)
{
if (controlFile.equals(file))
{
continue;
}
// See if it was handled already
if (processedDataSetFiles.contains(file))
{
continue;
}
unprcessedDataSets.add(file);
}
return unprcessedDataSets;
}
}
datastore_server/source/java/ch/systemsx/cisd/etlserver/entityregistration/SampleAndDataSetRegistrationGlobalState.java
+ 18
− 1
View file @ 41f0d020
......@@ -58,6 +58,10 @@ class SampleAndDataSetRegistrationGlobalState
private final String controlFilePattern;
private final boolean alwaysCleanupAfterProcessing;
private final boolean unmentionedSubfolderIsFailure;
private final Logger operationLog;
private IMailClient mailClient;
......@@ -66,7 +70,8 @@ class SampleAndDataSetRegistrationGlobalState
IEncapsulatedOpenBISService openbisService, SpaceIdentifier spaceIdentifierOrNull,
SampleType sampleTypeOrNull, DataSetType dataSetTypeOrNull,
SampleRegistrationMode sampleRegistrationMode, List<String> errorEmailRecipientsOrNull,
String controlFilePattern, Logger operationLog)
String controlFilePattern, boolean alwaysCleanupAfterProcessing,
boolean unmentionedSubfolderIsFailure, Logger operationLog)
{
this.delegator = delegator;
this.openbisService = openbisService;
......@@ -76,6 +81,8 @@ class SampleAndDataSetRegistrationGlobalState
this.sampleRegistrationMode = sampleRegistrationMode;
this.errorEmailRecipientsOrNull = errorEmailRecipientsOrNull;
this.controlFilePattern = controlFilePattern;
this.alwaysCleanupAfterProcessing = alwaysCleanupAfterProcessing;
this.unmentionedSubfolderIsFailure = unmentionedSubfolderIsFailure;
this.operationLog = operationLog;
}
......@@ -174,4 +181,14 @@ class SampleAndDataSetRegistrationGlobalState
{
return controlFilePattern;
}
public boolean alwaysCleanUpAfterProcessing()
{
return alwaysCleanupAfterProcessing;
}
public boolean areUnmentionedFoldersAnError()
{
return unmentionedSubfolderIsFailure;
}
}
datastore_server/source/java/ch/systemsx/cisd/etlserver/entityregistration/SampleAndDataSetRegistrationHandler.java
+ 22
− 4
View file @ 41f0d020
......@@ -67,7 +67,13 @@ public class SampleAndDataSetRegistrationHandler implements IDataSetHandlerWithM
static final String CONTROL_FILE_REGEX_PATTERN = "control-file-regex-pattern";
private static final String DEFAULT_CONTROL_FILE_REGEX_PATTERN = ".*.[Tt][Ss][Vv]";
private static final String DEFAULT_CONTROL_FILE_REGEX_PATTERN = ".*\\.[Tt][Ss][Vv]";
static final String CONTROL_FILE_ALWAYS_CLEANUP_AFTER_PROCESSING =
"always-cleanup-after-processing";
static final String CONTROL_FILE_UNMENTIONED_SUBFOLDER_IS_FAILURE =
"unmentioned-subfolder-is-failure";
private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
SampleAndDataSetRegistrationHandler.class);
......@@ -134,9 +140,18 @@ public class SampleAndDataSetRegistrationHandler implements IDataSetHandlerWithM
PropertyUtils.getProperty(specificProperties, CONTROL_FILE_REGEX_PATTERN,
DEFAULT_CONTROL_FILE_REGEX_PATTERN);
boolean deleteFilesOnFailure =
PropertyUtils.getBoolean(specificProperties,
CONTROL_FILE_ALWAYS_CLEANUP_AFTER_PROCESSING, true);
boolean unmentionedSubfolderIsFailure =
PropertyUtils.getBoolean(specificProperties,
CONTROL_FILE_UNMENTIONED_SUBFOLDER_IS_FAILURE, true);
return new SampleAndDataSetRegistrationGlobalState(delegator, service, spaceIdentifier,
sampleTypeOrNull, dataSetTypeOrNull, registrationMode, errorEmailRecipients,
controlFilePattern, operationLog);
controlFilePattern, deleteFilesOnFailure, unmentionedSubfolderIsFailure,
operationLog);
}
public void initializeMailClient(IMailClient mailClient)
......@@ -158,8 +173,11 @@ public class SampleAndDataSetRegistrationHandler implements IDataSetHandlerWithM
throw new CheckedExceptionTunnel(ex);
} finally
{
FileOperations.getMonitoredInstanceForCurrentThread().deleteRecursively(file);
logFileDeletion(file);
if (globalState.alwaysCleanUpAfterProcessing())
{
FileOperations.getMonitoredInstanceForCurrentThread().deleteRecursively(file);
logFileDeletion(file);
}
}
return createReturnValue();
}
......
datastore_server/source/java/ch/systemsx/cisd/etlserver/entityregistration/SampleAndDataSetRegistrator.java
+ 28
− 49
View file @ 41f0d020
......@@ -40,57 +40,21 @@ class SampleAndDataSetRegistrator extends AbstractSampleAndDataSetProcessor impl
TransferredDataSetHandler.IDataSetRegistrator
{
/**
* A wrapper around errors encountered during registration.
*
* @author Chandrasekhar Ramakrishnan
*/
static class RegistrationErrorWrapper
{
private final boolean isError;
private final Throwable errorOrNull;
RegistrationErrorWrapper(Throwable errorOrNull)
{
this.errorOrNull = errorOrNull;
isError = errorOrNull != null;
}
public boolean isError()
{
return isError;
}
public Throwable getErrorOrNull()
{
return errorOrNull;
}
public String getMessage()
{
if (isError)
{
return errorOrNull.getMessage();
} else
{
return "Success";
}
}
}
private final ControlFileRegistrationProperties properties;
private final SampleDataSetPair sampleDataSetPair;
// State that is updated during the registration
// The file/folder to register
private File dataSetFile;
// The sample, if the sample exists, null otherwise.
private Sample sampleOrNull;
private boolean didSucceed = false;
private RegistrationErrorWrapper failureException = null;
private IRegistrationStatus failureException = null;
SampleAndDataSetRegistrator(File folder, ControlFileRegistrationProperties properties,
SampleDataSetPair sampleDataSetPair)
......@@ -105,18 +69,18 @@ class SampleAndDataSetRegistrator extends AbstractSampleAndDataSetProcessor impl
*
* @return An exception if one was encountered, otherwise null.
*/
public RegistrationErrorWrapper register()
public IRegistrationStatus register()
{
File dataSetFile = new File(folder, sampleDataSetPair.getFolderName());
dataSetFile = new File(folder, sampleDataSetPair.getFolderName());
try
{
checkDataSetFileNotEmpty(dataSetFile);
checkDataSetFileNotEmpty();
checkExperimentExists();
retrieveSampleOrNull();
checkConformanceToMode();
} catch (UserFailureException ex)
{
return new RegistrationErrorWrapper(ex);
return new RegistrationError(ex);
}
if (globalState.getDelegator() instanceof IExtensibleDataSetHandler)
......@@ -129,7 +93,22 @@ class SampleAndDataSetRegistrator extends AbstractSampleAndDataSetProcessor impl
logDataRegistered();
}
}
return failureException;
if (null == failureException)
{
return new RegistrationSuccess(sampleDataSetPair.getTokens());
} else
{
return failureException;
}
}
/**
* Return the file that we treat as a data set. This is only valid *after* register has been
* called.
*/
public File getDataSetFile()
{
return dataSetFile;
}
/**
......@@ -159,7 +138,7 @@ class SampleAndDataSetRegistrator extends AbstractSampleAndDataSetProcessor impl
}
}
private void checkDataSetFileNotEmpty(File dataSetFile)
private void checkDataSetFileNotEmpty()
{
if (0 == dataSetFile.list().length)
{
......@@ -239,13 +218,13 @@ class SampleAndDataSetRegistrator extends AbstractSampleAndDataSetProcessor impl
} catch (UserFailureException e)
{
didSucceed = false;
failureException = new RegistrationErrorWrapper(e);
failureException = new RegistrationError(e);
throw e;
} catch (Throwable e)
{
logError("Could not register " + sampleDataSetPair + " in openBIS", e);
didSucceed = false;
failureException = new RegistrationErrorWrapper(e);
failureException = new RegistrationError(e);
throw e;
}
}
......
datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/entityregistration/test-data/not-all-subfolders-mentioned/control.tsv
+ 1
− 2
View file @ 41f0d020
......@@ -7,5 +7,4 @@
# Data
S_identifier S_experiment S_PROP1 S_PROP2 S_PROP3 D_file_type D_PROP1 D_PROP2 FOLDER
/MYSPACE/S1 /MYSPACE/MYPROJ/EXP1 VAL10 VAL20 VAL30 FILE_TYPE VAL40 VAL50 ds1/
/MYSPACE/S2 /MYSPACE/MYPROJ/EXP2 VAL11 VAL21 VAL31 FILE_TYPE VAL41 VAL51 ds2/
/MYSPACE/S3 /MYSPACE/MYPROJ/EXP3 VAL12 VAL22 VAL32 FILE_TYPE VAL42 VAL52 ds3/
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment