diff --git a/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/LinkMakerProvider.java b/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/LinkMakerProvider.java
index 296e9cee8d49e1e45e872ca6eb9a63e09cdf3732..9944210aa2f3d5b025168cda5a3f8edb017cb748 100644
--- a/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/LinkMakerProvider.java
+++ b/bds/source/java/ch/systemsx/cisd/bds/storage/filesystem/LinkMakerProvider.java
@@ -16,7 +16,7 @@
package ch.systemsx.cisd.bds.storage.filesystem;
-import ch.systemsx.cisd.common.Constants;
+import ch.systemsx.cisd.common.TimingParameters;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.utilities.FastRecursiveHardLinkMaker;
import ch.systemsx.cisd.common.utilities.IImmutableCopier;
@@ -31,8 +31,6 @@ public final class LinkMakerProvider
private static final String NO_HARD_LINK_EXECUTABLE = "No hard link executable has been found.";
- private static final int MAX_COPY_RETRIES = 7;
-
private static IImmutableCopier hardLinkMaker;
private LinkMakerProvider()
@@ -43,8 +41,7 @@ public final class LinkMakerProvider
private final static IImmutableCopier tryCreateHardLinkMaker()
{
final IImmutableCopier copier =
- FastRecursiveHardLinkMaker.tryCreate(Constants.MILLIS_TO_WAIT_BEFORE_TIMEOUT,
- MAX_COPY_RETRIES, Constants.MILLIS_TO_SLEEP_BEFORE_RETRYING);
+ FastRecursiveHardLinkMaker.tryCreate(TimingParameters.getDefaultParameters());
if (copier != null)
{
return copier;
diff --git a/common/source/java/ch/systemsx/cisd/common/Constants.java b/common/source/java/ch/systemsx/cisd/common/Constants.java
index 31b7c2cdec3e98cea17e0ab8480003b94346ceac..a3ec8610c744c0830b910eb7e2c013893b9f885a 100644
--- a/common/source/java/ch/systemsx/cisd/common/Constants.java
+++ b/common/source/java/ch/systemsx/cisd/common/Constants.java
@@ -41,19 +41,31 @@ public final class Constants
public static final String DELETION_IN_PROGRESS_PREFIX =
MARKER_PREFIX + "deletion_in_progress_";
- /** The number of milliseconds to wait before considering timeout (<i>10s</i>). */
- public static final long MILLIS_TO_WAIT_BEFORE_TIMEOUT = 10 * DateUtils.MILLIS_PER_SECOND;
-
- /** The number of milliseconds to sleep before retrying (<i>3s</i>). */
- public static final long MILLIS_TO_SLEEP_BEFORE_RETRYING = 3 * DateUtils.MILLIS_PER_SECOND;
-
- /** The maximal number of invocations when a given process failed. */
- public static final int MAXIMUM_INVOCATIONS_ON_FAILURE = 12;
-
/** The prefix of marker files that indicate that a directory is currently being processed. */
public static final String PROCESSING_PREFIX = MARKER_PREFIX + "processing_";
/** The file name of the file that contains file names which are known to be bad. */
public static final String FAULTY_PATH_FILENAME = ".faulty_paths";
+ //
+ // Timing
+ //
+
+ /** Time in seconds to wait before a timeout occurs. */
+ public static final int TIMEOUT_SECONDS = 60;
+
+ /** Time in milli-seconds to wait before a timeout occurs.. */
+ public static final long MILLIS_TO_WAIT_BEFORE_TIMEOUT =
+ TIMEOUT_SECONDS * DateUtils.MILLIS_PER_SECOND;
+
+ /** Time interval in seconds to wait after an operation failed before retrying it. */
+ public static final int INTERVAL_TO_WAIT_AFTER_FAILURE_SECONDS = 10;
+
+ /** Time interval in milli-seconds to wait after an operation failed before retrying it. */
+ public static final long MILLIS_TO_SLEEP_BEFORE_RETRYING =
+ INTERVAL_TO_WAIT_AFTER_FAILURE_SECONDS * DateUtils.MILLIS_PER_SECOND;
+
+ /** Maximal number of retries when an operation fails. */
+ public static final int MAXIMUM_RETRY_COUNT = 11;
+
}
diff --git a/common/source/java/ch/systemsx/cisd/common/TimingParameters.java b/common/source/java/ch/systemsx/cisd/common/TimingParameters.java
new file mode 100644
index 0000000000000000000000000000000000000000..18935b6cde40c21274049886991fd09fee3361bd
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/TimingParameters.java
@@ -0,0 +1,258 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common;
+
+import java.util.Properties;
+
+import org.apache.commons.lang.builder.EqualsBuilder;
+import org.apache.commons.lang.builder.HashCodeBuilder;
+import org.apache.commons.lang.time.DateUtils;
+
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+
+/**
+ * A class that provides timing parameters for operations that can timeout or otherwise fail.
+ *
+ * @author Bernd Rinn
+ */
+public class TimingParameters
+{
+
+ private final static TimingParameters standardParameters =
+ new TimingParameters(Constants.MILLIS_TO_WAIT_BEFORE_TIMEOUT,
+ Constants.MAXIMUM_RETRY_COUNT, Constants.MILLIS_TO_SLEEP_BEFORE_RETRYING);
+
+ private final static TimingParameters defaultParameters =
+ new TimingParameters(standardParameters);
+
+ private final static TimingParameters noTimeoutNoRetriesParameters =
+ new TimingParameters(Long.MAX_VALUE, 0, 0L);
+
+ /**
+ * Returns the standard timing parameters (may not be overwritten).
+ */
+ public static final TimingParameters getStandardParameters()
+ {
+ return standardParameters;
+ }
+
+ /**
+ * Returns the default timing parameters.
+ */
+ public static final TimingParameters getDefaultParameters()
+ {
+ return defaultParameters;
+ }
+
+ /**
+ * Returns timing parameters that corresponds to 'no timeout', 'no retries'.
+ */
+ public static final TimingParameters getNoTimeoutNoRetriesParameters()
+ {
+ return noTimeoutNoRetriesParameters;
+ }
+
+ public static final String TIMEOUT_PROPERTY = "timeout";
+
+ public static final String MAX_RETRY_PROPERTY = "max-retries";
+
+ public static final String FAILURE_INTERVAL = "failure-interval";
+
+ private volatile long timeoutMillis;
+
+ private volatile int maxRetriesOnFailure;
+
+ private volatile long intervalToWaitAfterFailureMillis;
+
+ public static TimingParameters create(final Properties properties)
+ {
+ return new TimingParameters(properties);
+ }
+
+ /**
+ * Returns <code>true</code> if the <var>properties</var> contain timing parameters and
+ * <code>false</code> otherwise.
+ */
+ public static boolean hasTimingParameters(Properties properties)
+ {
+ return properties.containsKey(TIMEOUT_PROPERTY)
+ || properties.containsKey(MAX_RETRY_PROPERTY)
+ || properties.containsKey(FAILURE_INTERVAL);
+ }
+
+ /**
+ * Sets the new default.
+ */
+ public static void setDefault(final long timeoutMillis, final int maxRetriesOnFailure,
+ final long intervalToWaitAfterFailureMillis)
+ {
+ defaultParameters.timeoutMillis = timeoutMillis;
+ defaultParameters.maxRetriesOnFailure = maxRetriesOnFailure;
+ defaultParameters.intervalToWaitAfterFailureMillis = intervalToWaitAfterFailureMillis;
+ }
+
+ /**
+ * Sets the new default.
+ */
+ public static void setDefault(final TimingParameters timingParameters)
+ {
+ defaultParameters.timeoutMillis = timingParameters.timeoutMillis;
+ defaultParameters.maxRetriesOnFailure = timingParameters.maxRetriesOnFailure;
+ defaultParameters.intervalToWaitAfterFailureMillis =
+ timingParameters.intervalToWaitAfterFailureMillis;
+ }
+
+ /**
+ * Sets the new default from the properties (or fall back to standard values, if
+ * <var>properties</var> do not contain values).
+ */
+ public static void setDefault(Properties properties)
+ {
+ defaultParameters.timeoutMillis = getTimeoutMillis(properties);
+ defaultParameters.maxRetriesOnFailure = getMaxRetriesOnFailure(properties);
+ defaultParameters.intervalToWaitAfterFailureMillis =
+ getIntervalToWaitAfterFailureMillis(properties);
+ }
+
+ private static long getIntervalToWaitAfterFailureMillis(Properties properties)
+ {
+ return PropertyUtils.getInt(properties, FAILURE_INTERVAL,
+ Constants.INTERVAL_TO_WAIT_AFTER_FAILURE_SECONDS)
+ * DateUtils.MILLIS_PER_SECOND;
+ }
+
+ private static int getMaxRetriesOnFailure(Properties properties)
+ {
+ return PropertyUtils.getInt(properties, MAX_RETRY_PROPERTY, Constants.MAXIMUM_RETRY_COUNT);
+ }
+
+ private static long getTimeoutMillis(Properties properties)
+ {
+ final long timeoutSpecified =
+ PropertyUtils.getInt(properties, TIMEOUT_PROPERTY, Constants.TIMEOUT_SECONDS)
+ * DateUtils.MILLIS_PER_SECOND;
+ return (timeoutSpecified == 0) ? Long.MAX_VALUE : timeoutSpecified;
+ }
+
+ public static TimingParameters create(final long timeoutMillis, final int maxRetriesOnFailure,
+ final long intervalToWaitAfterFailureMillis)
+ {
+ return new TimingParameters(timeoutMillis, maxRetriesOnFailure,
+ intervalToWaitAfterFailureMillis);
+ }
+
+ public static TimingParameters create(final long timeoutMillis)
+ {
+ return new TimingParameters(timeoutMillis, Constants.MAXIMUM_RETRY_COUNT,
+ Constants.MILLIS_TO_SLEEP_BEFORE_RETRYING);
+ }
+
+ public static TimingParameters createNoRetries(final long timeoutMillis)
+ {
+ return new TimingParameters(timeoutMillis, 0, 0L);
+ }
+
+ private TimingParameters(TimingParameters parameters)
+ {
+ this.timeoutMillis = parameters.timeoutMillis;
+ this.maxRetriesOnFailure = parameters.maxRetriesOnFailure;
+ this.intervalToWaitAfterFailureMillis = parameters.intervalToWaitAfterFailureMillis;
+ }
+
+ private TimingParameters(final long timeoutMillis, final int maxRetriesOnFailure,
+ final long intervalToWaitAfterFailureMillis)
+ {
+ this.timeoutMillis = timeoutMillis;
+ this.maxRetriesOnFailure = maxRetriesOnFailure;
+ this.intervalToWaitAfterFailureMillis = intervalToWaitAfterFailureMillis;
+ }
+
+ private TimingParameters(Properties properties)
+ {
+ this.timeoutMillis = getTimeoutMillis(properties);
+ this.maxRetriesOnFailure = getMaxRetriesOnFailure(properties);
+ this.intervalToWaitAfterFailureMillis = getIntervalToWaitAfterFailureMillis(properties);
+ }
+
+ /**
+ * Returns the timeout (in milli-seconds).
+ */
+ public long getTimeoutMillis()
+ {
+ return timeoutMillis;
+ }
+
+ /**
+ * Returns the maximal number of retries when an operation fails.
+ */
+ public int getMaxRetriesOnFailure()
+ {
+ return maxRetriesOnFailure;
+ }
+
+ /**
+ * Returns the time (in milli-seconds) to wait after a failure has occurred and before retrying
+ * the operation.
+ */
+ public long getIntervalToWaitAfterFailureMillis()
+ {
+ return intervalToWaitAfterFailureMillis;
+ }
+
+ //
+ // Object
+ //
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ if (obj == this)
+ {
+ return true;
+ }
+ if (obj instanceof TimingParameters == false)
+ {
+ return false;
+ }
+ final TimingParameters that = (TimingParameters) obj;
+ final EqualsBuilder builder = new EqualsBuilder();
+ builder.append(this.timeoutMillis, that.timeoutMillis);
+ builder.append(this.maxRetriesOnFailure, that.maxRetriesOnFailure);
+ builder
+ .append(this.intervalToWaitAfterFailureMillis,
+ that.intervalToWaitAfterFailureMillis);
+ return builder.isEquals();
+ }
+
+ @Override
+ public int hashCode()
+ {
+ final HashCodeBuilder builder = new HashCodeBuilder();
+ builder.append(timeoutMillis);
+ builder.append(maxRetriesOnFailure);
+ builder.append(intervalToWaitAfterFailureMillis);
+ return builder.toHashCode();
+ }
+
+ @Override
+ public String toString()
+ {
+ return String.format("Timing: timeout: %d s, maximal retries: %d, sleep on failure: %d s",
+ timeoutMillis / 1000, maxRetriesOnFailure, intervalToWaitAfterFailureMillis / 1000);
+ }
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/concurrent/ConcurrencyUtilities.java b/common/source/java/ch/systemsx/cisd/common/concurrent/ConcurrencyUtilities.java
index 464b8e3e8512b4305237d3b857d7a41ed53903ba..659f8093390f0ffd12770d24b9275ae10f991c1a 100644
--- a/common/source/java/ch/systemsx/cisd/common/concurrent/ConcurrencyUtilities.java
+++ b/common/source/java/ch/systemsx/cisd/common/concurrent/ConcurrencyUtilities.java
@@ -43,6 +43,27 @@ public final class ConcurrencyUtilities
/** Corresponds to an immediate timeout when waiting for the future. */
public static final long IMMEDIATE_TIMEOUT = 0L;
+ /**
+ * Provides settings for error logging.
+ */
+ public interface ILogSettings
+ {
+ /**
+ * Returns the logger to be used for errors.
+ */
+ ISimpleLogger getLogger();
+
+ /**
+ * Returns the operation name to be used in error logging.
+ */
+ String getOperationName();
+
+ /**
+ * Returns the log level to be used in error logging.
+ */
+ LogLevel getLogLevelForError();
+ }
+
/**
* Tries to get the result of a <var>future</var>, maximally waiting <var>timeoutMillis</var>
* for the result to become available. Any {@link ExecutionException} that might occur in the
@@ -57,7 +78,7 @@ public final class ConcurrencyUtilities
*/
public static <T> T tryGetResult(Future<T> future, long timeoutMillis) throws StopException
{
- return tryGetResult(future, timeoutMillis, null, null, true);
+ return tryGetResult(future, timeoutMillis, null, true);
}
/**
@@ -76,7 +97,7 @@ public final class ConcurrencyUtilities
public static <T> T tryGetResult(Future<T> future, long timeoutMillis, boolean stopOnInterrupt)
throws StopException
{
- return tryGetResult(future, timeoutMillis, null, null, stopOnInterrupt);
+ return tryGetResult(future, timeoutMillis, null, stopOnInterrupt);
}
/**
@@ -88,12 +109,10 @@ public final class ConcurrencyUtilities
* @param future The future representing the execution to wait for.
* @param timeoutMillis The time-out (in milliseconds) to wait for the execution to finish. If
* it is smaller than 0, no time-out will apply.
- * @param loggerOrNull The logger to use for logging note-worthy information, or
- * <code>null</code>, if nothing should be logged.
- * @param operationNameOrNull The name of the operation performed, for log messages, or
- * <code>null</code>, if it is not known or deemed unimportant.
- * @param stopOnInterrupt If <code>true</code>, throw a {@link StopException} if the thread
- * gets interrupted while waiting on the future.
+ * @param logSettingsOrNull The settings for error logging, or <code>null</code>, if error
+ * conditions should not be logged.
+ * @param stopOnInterrupt If <code>true</code>, throw a {@link StopException} if the thread gets
+ * interrupted while waiting on the future.
* @return The result of the future, or <code>null</code>, if the result does not become
* available within <var>timeoutMillis</var> ms or if the waiting thread gets
* interrupted.
@@ -101,11 +120,9 @@ public final class ConcurrencyUtilities
* <code>true</code>.
*/
public static <T> T tryGetResult(Future<T> future, long timeoutMillis,
- ISimpleLogger loggerOrNull, String operationNameOrNull, boolean stopOnInterrupt)
- throws StopException
+ ILogSettings logSettingsOrNull, boolean stopOnInterrupt) throws StopException
{
- final ExecutionResult<T> result =
- getResult(future, timeoutMillis, loggerOrNull, operationNameOrNull);
+ final ExecutionResult<T> result = getResult(future, timeoutMillis, logSettingsOrNull);
return tryDealWithResult(result, stopOnInterrupt);
}
@@ -129,12 +146,11 @@ public final class ConcurrencyUtilities
* with the deviant cases yourself, then call this method to deal with the rest.
*
* @param result A
- * @param stopOnInterrupt If <code>true</code>, throw a {@link StopException} if the thread
- * gets interrupted while waiting on the future.
- * @return The value of the <var>result</var> of the future, or <code>null</code>, if the
- * result status is {@link ExecutionStatus#TIMED_OUT} or
- * {@link ExecutionStatus#INTERRUPTED} and <var>stopOnInterrupt</var> is
- * <code>false</code>.
+ * @param stopOnInterrupt If <code>true</code>, throw a {@link StopException} if the thread gets
+ * interrupted while waiting on the future.
+ * @return The value of the <var>result</var> of the future, or <code>null</code>, if the result
+ * status is {@link ExecutionStatus#TIMED_OUT} or {@link ExecutionStatus#INTERRUPTED}
+ * and <var>stopOnInterrupt</var> is <code>false</code>.
* @throws StopException If the thread got interrupted and <var>stopOnInterrupt</var> is
* <code>true</code>.
* @throws RuntimeException If the result status is {@link ExecutionStatus#EXCEPTION} and the
@@ -180,17 +196,16 @@ public final class ConcurrencyUtilities
}
/**
- * Returns the result of a <var>future</var>, maximally waiting <var>timeoutMillis</var> for
- * the result to become available. The return value is never <code>null</code>, but always a
+ * Returns the result of a <var>future</var>, maximally waiting <var>timeoutMillis</var> for the
+ * result to become available. The return value is never <code>null</code>, but always a
* {@link ExecutionResult} that describes the outcome of the execution. The possible outcomes
* are:
* <ul>
* <li> {@link ExecutionStatus#COMPLETE}: The execution has been performed correctly and a
- * result is available, if provided.</li>
- * <li> {@link ExecutionStatus#EXCEPTION}: The execution has been terminated by an exception.</li>
- * <li> {@link ExecutionStatus#TIMED_OUT}: The execution timed out.</li>
- * <li> {@link ExecutionStatus#INTERRUPTED}: The thread of the execution was interrupted (see
- * {@link Thread#interrupt()}).</li>
+ * result is available, if provided.</li> <li> {@link ExecutionStatus#EXCEPTION}: The execution
+ * has been terminated by an exception.</li> <li> {@link ExecutionStatus#TIMED_OUT}: The
+ * execution timed out.</li> <li> {@link ExecutionStatus#INTERRUPTED}: The thread of the
+ * execution was interrupted (see {@link Thread#interrupt()}).</li>
* </ul>
*
* @param future The future representing the execution to wait for.
@@ -201,51 +216,81 @@ public final class ConcurrencyUtilities
*/
public static <T> ExecutionResult<T> getResult(Future<T> future, long timeoutMillis)
{
- return getResult(future, timeoutMillis, null, null);
+ return getResult(future, timeoutMillis, null);
+ }
+
+ /**
+ * Returns the result of a <var>future</var>, maximally waiting <var>timeoutMillis</var> for the
+ * result to become available. The return value is never <code>null</code>, but always a
+ * {@link ExecutionResult} that describes the outcome of the execution. The possible outcomes
+ * are:
+ * <ul>
+ * <li> {@link ExecutionStatus#COMPLETE}: The execution has been performed correctly and a
+ * result is available, if provided.</li> <li> {@link ExecutionStatus#EXCEPTION}: The execution
+ * has been terminated by an exception.</li> <li> {@link ExecutionStatus#TIMED_OUT}: The
+ * execution timed out.</li> <li> {@link ExecutionStatus#INTERRUPTED}: The thread of the
+ * execution was interrupted (see {@link Thread#interrupt()}).</li>
+ * </ul>
+ *
+ * @param future The future representing the execution to wait for.
+ * @param timeoutMillis The time-out (in milliseconds) to wait for the execution to finish. If
+ * it is smaller than 0, no time-out will apply.
+ * @param logSettingsOrNull The settings for error logging, or <code>null</code>, if error
+ * conditions should not be logged.
+ * @return The {@link ExecutionResult} of the <var>future</var>. May correspond to each one of
+ * the {@link ExecutionStatus} values.
+ */
+ public static <T> ExecutionResult<T> getResult(Future<T> future, long timeoutMillis,
+ ILogSettings logSettingsOrNull)
+ {
+ return getResult(future, timeoutMillis, true, logSettingsOrNull);
}
/**
- * Returns the result of a <var>future</var>, maximally waiting <var>timeoutMillis</var> for
- * the result to become available. The return value is never <code>null</code>, but always a
+ * Returns the result of a <var>future</var>, at most waiting <var>timeoutMillis</var> for the
+ * result to become available. The return value is never <code>null</code>, but always a
* {@link ExecutionResult} that describes the outcome of the execution. The possible outcomes
* are:
* <ul>
* <li> {@link ExecutionStatus#COMPLETE}: The execution has been performed correctly and a
- * result is available, if provided.</li>
- * <li> {@link ExecutionStatus#EXCEPTION}: The execution has been terminated by an exception.</li>
- * <li> {@link ExecutionStatus#TIMED_OUT}: The execution timed out.</li>
- * <li> {@link ExecutionStatus#INTERRUPTED}: The thread of the execution was interrupted (see
- * {@link Thread#interrupt()}).</li>
+ * result is available, if provided.</li> <li> {@link ExecutionStatus#EXCEPTION}: The execution
+ * has been terminated by an exception.</li> <li> {@link ExecutionStatus#TIMED_OUT}: The
+ * execution timed out.</li> <li> {@link ExecutionStatus#INTERRUPTED}: The thread of the
+ * execution was interrupted (see {@link Thread#interrupt()}).</li>
* </ul>
*
* @param future The future representing the execution to wait for.
* @param timeoutMillis The time-out (in milliseconds) to wait for the execution to finish. If
* it is smaller than 0, no time-out will apply.
- * @param loggerOrNull The logger to use for logging note-worthy information, or
- * <code>null</code>, if nothing should be logged.
- * @param operationNameOrNull The name of the operation performed, for log messages, or
- * <code>null</code>, if it is not known or deemed unimportant.
+ * @param cancelOnTimeout If <code>true</code>, the <var>future</var> will be canceled on
+ * time-out.
+ * @param logSettingsOrNull The settings for error logging, or <code>null</code>, if error
+ * conditions should not be logged.
* @return The {@link ExecutionResult} of the <var>future</var>. May correspond to each one of
* the {@link ExecutionStatus} values.
*/
public static <T> ExecutionResult<T> getResult(Future<T> future, long timeoutMillis,
- ISimpleLogger loggerOrNull, String operationNameOrNull)
+ boolean cancelOnTimeout, ILogSettings logSettingsOrNull)
+ {
+ return getResult(future, timeoutMillis, cancelOnTimeout, logSettingsOrNull, null);
+ }
+
+ private static boolean isActive(IActivitySensor sensorOrNull, long timeoutMillis)
{
- return getResult(future, timeoutMillis, true, loggerOrNull, operationNameOrNull);
+ return (sensorOrNull != null) && sensorOrNull.hasActivityMoreRecentThan(timeoutMillis);
}
/**
- * Returns the result of a <var>future</var>, maximally waiting <var>timeoutMillis</var> for
- * the result to become available. The return value is never <code>null</code>, but always a
+ * Returns the result of a <var>future</var>, at most waiting <var>timeoutMillis</var> for the
+ * result to become available. The return value is never <code>null</code>, but always a
* {@link ExecutionResult} that describes the outcome of the execution. The possible outcomes
* are:
* <ul>
* <li> {@link ExecutionStatus#COMPLETE}: The execution has been performed correctly and a
- * result is available, if provided.</li>
- * <li> {@link ExecutionStatus#EXCEPTION}: The execution has been terminated by an exception.</li>
- * <li> {@link ExecutionStatus#TIMED_OUT}: The execution timed out.</li>
- * <li> {@link ExecutionStatus#INTERRUPTED}: The thread of the execution was interrupted (see
- * {@link Thread#interrupt()}).</li>
+ * result is available, if provided.</li> <li> {@link ExecutionStatus#EXCEPTION}: The execution
+ * has been terminated by an exception.</li> <li> {@link ExecutionStatus#TIMED_OUT}: The
+ * execution timed out.</li> <li> {@link ExecutionStatus#INTERRUPTED}: The thread of the
+ * execution was interrupted (see {@link Thread#interrupt()}).</li>
* </ul>
*
* @param future The future representing the execution to wait for.
@@ -253,65 +298,98 @@ public final class ConcurrencyUtilities
* it is smaller than 0, no time-out will apply.
* @param cancelOnTimeout If <code>true</code>, the <var>future</var> will be canceled on
* time-out.
- * @param loggerOrNull The logger to use for logging note-worthy information, or
- * <code>null</code>, if nothing should be logged.
- * @param operationNameOrNull The name of the operation performed, for log messages, or
- * <code>null</code>, if it is not known or deemed unimportant.
+ * @param logSettingsOrNull The settings for error logging, or <code>null</code>, if error
+ * conditions should not be logged.
+ * @param sensorOrNull A sensor that can prevent the method from timing out by showing activity.
* @return The {@link ExecutionResult} of the <var>future</var>. May correspond to each one of
* the {@link ExecutionStatus} values.
*/
public static <T> ExecutionResult<T> getResult(Future<T> future, long timeoutMillis,
- boolean cancelOnTimeout, ISimpleLogger loggerOrNull, String operationNameOrNull)
+ boolean cancelOnTimeout, ILogSettings logSettingsOrNull, IActivitySensor sensorOrNull)
{
- final String operationName =
- (operationNameOrNull == null) ? "UNKNOWN" : operationNameOrNull;
try
{
- return ExecutionResult.create(future.get(transform(timeoutMillis),
- TimeUnit.MILLISECONDS));
- } catch (TimeoutException ex)
+ ExecutionResult<T> result = null;
+ do
+ {
+ try
+ {
+ result = ExecutionResult.create(future.get(transform(timeoutMillis),
+ TimeUnit.MILLISECONDS));
+ } catch (TimeoutException ex)
+ {
+ // result is still null
+ }
+ } while (result == null && isActive(sensorOrNull, timeoutMillis));
+ if (result == null)
+ {
+ if (cancelOnTimeout)
+ {
+ future.cancel(true);
+ }
+ if (logSettingsOrNull != null)
+ {
+ logSettingsOrNull.getLogger().log(
+ logSettingsOrNull.getLogLevelForError(),
+ String.format("%s: timeout of %.2f s exceeded%s.", logSettingsOrNull
+ .getOperationName(), timeoutMillis / 1000f,
+ cancelOnTimeout ? ", cancelled" : ""));
+ }
+ return ExecutionResult.createTimedOut();
+ } else
+ {
+ return result;
+ }
+ } catch (InterruptedException ex)
{
- if (cancelOnTimeout)
+ future.cancel(true);
+ if (logSettingsOrNull != null)
{
- future.cancel(true);
+ logSettingsOrNull.getLogger().log(logSettingsOrNull.getLogLevelForError(),
+ String.format("%s: interrupted.", logSettingsOrNull.getOperationName()));
}
- if (loggerOrNull != null)
+ return ExecutionResult.createInterrupted();
+ } catch (StopException ex)
+ {
+ // Happens when Thread.stop(new StopException()) is called.
+ future.cancel(true);
+ if (logSettingsOrNull != null)
{
- loggerOrNull.log(LogLevel.DEBUG, String.format(
- "%s took longer than %f s, cancelled.", operationName,
- timeoutMillis / 1000f));
+ logSettingsOrNull.getLogger().log(logSettingsOrNull.getLogLevelForError(),
+ String.format("%s: stopped.", logSettingsOrNull.getOperationName()));
}
- return ExecutionResult.createTimedOut();
- } catch (InterruptedException ex)
+ return ExecutionResult.createInterrupted();
+ } catch (ThreadDeath ex)
{
future.cancel(true);
- if (loggerOrNull != null)
+ if (logSettingsOrNull != null)
{
- loggerOrNull.log(LogLevel.DEBUG, String
- .format("%s got interrupted.", operationName));
+ logSettingsOrNull.getLogger().log(logSettingsOrNull.getLogLevelForError(),
+ String.format("%s: stopped.", logSettingsOrNull.getOperationName()));
}
return ExecutionResult.createInterrupted();
} catch (CancellationException ex)
{
- if (loggerOrNull != null)
+ if (logSettingsOrNull != null)
{
- loggerOrNull.log(LogLevel.DEBUG, String
- .format("%s got cancelled.", operationName));
+ logSettingsOrNull.getLogger().log(logSettingsOrNull.getLogLevelForError(),
+ String.format("%s: cancelled.", logSettingsOrNull.getOperationName()));
}
- // We treat cancelled the same as interrupted.
return ExecutionResult.createInterrupted();
} catch (ExecutionException ex)
{
final Throwable cause = ex.getCause();
- if (loggerOrNull != null)
+ if (logSettingsOrNull != null)
{
final String message =
(cause == null || cause.getMessage() == null) ? "<no message>" : cause
.getMessage();
final String className =
(cause == null) ? "<unknown class>" : cause.getClass().getSimpleName();
- loggerOrNull.log(LogLevel.ERROR, String.format(
- "%s has caused an exception: %s [%s]", operationName, message, className));
+ logSettingsOrNull.getLogger().log(
+ logSettingsOrNull.getLogLevelForError(),
+ String.format("%s: exception: %s [%s].", logSettingsOrNull
+ .getOperationName(), message, className));
}
return ExecutionResult.createExceptional(cause == null ? ex : cause);
}
diff --git a/common/source/java/ch/systemsx/cisd/common/concurrent/IActivityObserver.java b/common/source/java/ch/systemsx/cisd/common/concurrent/IActivityObserver.java
new file mode 100644
index 0000000000000000000000000000000000000000..3713fa5820deb6740a82c9f12fb30b7ff014405a
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/concurrent/IActivityObserver.java
@@ -0,0 +1,32 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ch.systemsx.cisd.common.concurrent;
+
+import java.util.Observer;
+
+/**
+ * A simple role that can observe activity. It is a simple version of {@link Observer} that can only
+ * observes the fact that the {@link #update()} method has been called.
+ *
+ * @author Bernd Rinn
+ */
+public interface IActivityObserver
+{
+ /**
+ * Called when activity occurred.
+ */
+ public void update();
+}
\ No newline at end of file
diff --git a/common/source/java/ch/systemsx/cisd/common/concurrent/IActivitySensor.java b/common/source/java/ch/systemsx/cisd/common/concurrent/IActivitySensor.java
new file mode 100644
index 0000000000000000000000000000000000000000..ca32fe5a0c1cbbb1bfa2140d3f6f3c4f16877fee
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/concurrent/IActivitySensor.java
@@ -0,0 +1,47 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ch.systemsx.cisd.common.concurrent;
+
+/**
+ * A role that can provide activity information.
+ *
+ * @author Bernd Rinn
+ */
+public interface IActivitySensor
+{
+ /**
+ * Returns the time of last activity on this sensor in milli-seconds since start of the epoch.
+ * <p>
+ * Callers should expect that a call of this method may be expensive and thus only call it when
+ * necessary. However, the caller can expect the information returned by is to be up-to-date.
+ *
+ * @param thresholdMillis The time threshold for activity (in milli-seconds) that qualifies as
+ * "recent enough" to terminate the search for even more recent activity. An
+ * implementation can safely ignore this, as it is just meant for the sensor to
+ * optimize its search. A value <code>thresholdMillis=0</code> translates into
+ * "The absolutely most recent activity".
+ */
+ long getLastActivityMillisMoreRecentThan(long thresholdMillis);
+
+ /**
+ * Returns <code>true</code>, if this sensor has had activity more recent than
+ * <var>thresholdMillis</var> milli-seconds ago.
+ *
+ * @param thresholdMillis The time threshold for activity (in milli-seconds) that qualifies as
+ * "recent enough" to terminate the search for even more recent activity.
+ */
+ boolean hasActivityMoreRecentThan(long thresholdMillis);
+}
\ No newline at end of file
diff --git a/common/source/java/ch/systemsx/cisd/common/concurrent/InactivityMonitor.java b/common/source/java/ch/systemsx/cisd/common/concurrent/InactivityMonitor.java
index 194245ae4a7ba07daa54efa03bf85ed6ca5e8a9f..95863da93980275e8e86caf16735c947bd6bf0e9 100644
--- a/common/source/java/ch/systemsx/cisd/common/concurrent/InactivityMonitor.java
+++ b/common/source/java/ch/systemsx/cisd/common/concurrent/InactivityMonitor.java
@@ -40,26 +40,13 @@ public class InactivityMonitor
/**
* The sensor to get activity information from.
*/
- public interface IActivitySensor
+ public interface IDescribingActivitySensor extends IActivitySensor
{
- /**
- * Returns the time of last activity in milli-seconds since start of the epoch.
- * <p>
- * The monitor assumes that a call of this method may be expensive and thus only calls it
- * when necessary. However, the information returned is expected to be up-to-date.
- *
- * @param thresholdMillis The time threshold for activity (in milli-seconds) that qualifies
- * as "recent enough" to terminate the search for even more recent activity. An
- * implementation can safely ignore this, as it is just meant for the sensor to
- * optimize its search.
- */
- long getTimeOfLastActivityMoreRecentThan(long thresholdMillis);
-
/**
* Returns a string that describes the kind (and possibly reason) of recent inactivity.
* <p>
* Used for log messages. It can generally be assumed that this method is called after
- * {@link #getTimeOfLastActivityMoreRecentThan(long)}.
+ * {@link #getLastActivityMillisMoreRecentThan(long)}.
*
* @param now The current time as it should be used in the description.
*/
@@ -83,7 +70,7 @@ public class InactivityMonitor
void update(long inactiveSinceMillis, String descriptionOfInactivity);
}
- private final IActivitySensor sensor;
+ private final IDescribingActivitySensor sensor;
private final IInactivityObserver observer;
@@ -102,11 +89,11 @@ public class InactivityMonitor
* @param observer The observer to inform when the inactivity threshold has been exceeded.
* @param inactivityThresholdMillis The threshold of a period of inactivity that needs to be
* exceeded before the inactivity observer gets informed.
- * @param stopAfterFirstEvent If <code>true</code>, the monitor will stop itself after the
- * first event of exceeded inactivity threshold has happened, otherwise, the monitor
- * will continue to look for such events.
+ * @param stopAfterFirstEvent If <code>true</code>, the monitor will stop itself after the first
+ * event of exceeded inactivity threshold has happened, otherwise, the monitor will
+ * continue to look for such events.
*/
- public InactivityMonitor(IActivitySensor sensor, IInactivityObserver observer,
+ public InactivityMonitor(IDescribingActivitySensor sensor, IInactivityObserver observer,
long inactivityThresholdMillis, boolean stopAfterFirstEvent)
{
assert sensor != null;
@@ -162,7 +149,7 @@ public class InactivityMonitor
private void updateTimeOfActivity()
{
timeOfLastActivity =
- sensor.getTimeOfLastActivityMoreRecentThan(inactivityThresholdMillis);
+ sensor.getLastActivityMillisMoreRecentThan(inactivityThresholdMillis);
}
private boolean isInactivityThresholdExceeded(final long now)
diff --git a/common/source/java/ch/systemsx/cisd/common/concurrent/MonitoringProxy.java b/common/source/java/ch/systemsx/cisd/common/concurrent/MonitoringProxy.java
index 8135299e2e7d28b9bece1c04f9b1dedc9e6489db..e93916cf6af48f4569d13ab61426bb9156b897e6 100644
--- a/common/source/java/ch/systemsx/cisd/common/concurrent/MonitoringProxy.java
+++ b/common/source/java/ch/systemsx/cisd/common/concurrent/MonitoringProxy.java
@@ -20,22 +20,38 @@ import java.lang.reflect.InvocationHandler;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.lang.reflect.Proxy;
+import java.util.Collection;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.Map;
+import java.util.Set;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
+import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities.ILogSettings;
import ch.systemsx.cisd.common.exceptions.StopException;
import ch.systemsx.cisd.common.exceptions.TimeoutException;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.logging.LogLevel;
/**
* A class that can provide a dynamic {@link Proxy} for an interface that delegates the method
* invocations to an implementation of this interface, monitoring for calls to
- * {@link Thread#interrupt()} and for timeouts. (Note that by default no timeout is set.)
+ * {@link Thread#interrupt()} and for timeouts and allowing the invocation to be retried if it
+ * failed. (Note that by default no timeout is set and no retrying of failed operations is
+ * performed.)
* <p>
* On calls to {@link Thread#interrupt()} the proxy will throw a {@link StopException}, on timeouts
* a {@link TimeoutException}.
* <p>
+ * Retrying failed invocations is enabled by calling {@link #timing(TimingParameters)} with a retry
+ * parameter greater than 0. You need to carefully consider whether the methods in the interface are
+ * suitable for retrying or not. Note that there is no retrying on thread interruption, but only on
+ * timeout. You can configure a 'white-list' of exception classes that are suitable for retrying the
+ * operation. By default this white-list is empty, thus method invocations failing with an exception
+ * will not be retried.
+ * <p>
* The proxy can be configured by chaining. If all options have been set, the actual proxy can be
* obtained by calling {@link #get()}. In order to e.g. set the timeout to 10s, the following call
* chain can be used:
@@ -44,9 +60,9 @@ import ch.systemsx.cisd.common.exceptions.TimeoutException;
* If proxy = MonitoringProxy.create(If.class, someInstance).timeoutMillis(10000L).get();
* </pre>
*
- * Instead of throwing a exception, the proxy can also be configured to provide special error values
- * on error conditions. This configuration is done by {@link #errorValueOnInterrupt()} for thread
- * interrupts and {@link #errorValueOnTimeout()} for timeouts.
+ * Instead of throwing an exception, the proxy can also be configured to provide special error
+ * values on error conditions. This configuration is done by {@link #errorValueOnInterrupt()} for
+ * thread interrupts and {@link #errorValueOnTimeout()} for timeouts.
* <p>
* The error return values can be set, either for the type of the return value of a method, or by
* the method itself. If present, the specific method-value mapping will take precedence of the
@@ -58,14 +74,25 @@ import ch.systemsx.cisd.common.exceptions.TimeoutException;
* MonitoringProxy.create(If.class, someInstance).errorValueOnInterrupt()
* .errorTypeValueMapping(String.class, "ERROR").get();
* </pre>
+ * <p>
+ * <i>Note:</i> A MonitoringProxy object can only be used safely from more than one thread if
+ * <ol>
+ * <li>The proxied object is thread-safe
+ * <li>
+ * <li>No observer / sensor pattern is used to detect activity (can produce "false negatives" on
+ * hanging method calls)
+ * <li>
+ * </ol>
*
* @author Bernd Rinn
*/
public class MonitoringProxy<T>
{
+ private final static int NUMBER_OF_CORE_THREADS = 10;
private final static ExecutorService executor =
- new NamingThreadPoolExecutor("Monitoring Proxy").daemonize();
+ new NamingThreadPoolExecutor("Monitoring Proxy").corePoolSize(NUMBER_OF_CORE_THREADS)
+ .daemonize();
private final DelegatingInvocationHandler<T> delegate;
@@ -73,11 +100,13 @@ public class MonitoringProxy<T>
private final Map<Method, Object> errorMethodValueMap;
+ private final Set<Class<? extends Exception>> exceptionClassesSuitableForRetrying;
+
private final MonitoringInvocationHandler handler;
private final T proxy;
- private long timeoutMillis;
+ private TimingParameters timingParameters;
private boolean errorValueOnTimeout;
@@ -85,9 +114,12 @@ public class MonitoringProxy<T>
private String nameOrNull;
+ private ISimpleLogger loggerOrNull;
+
+ private IActivitySensor sensorOrNull;
+
private static class DelegatingInvocationHandler<T> implements InvocationHandler
{
-
private final T objectToProxyFor;
private DelegatingInvocationHandler(T objectToProxyFor)
@@ -96,10 +128,18 @@ public class MonitoringProxy<T>
}
public Object invoke(Object proxy, Method method, Object[] args) throws Throwable
+
{
try
{
- return method.invoke(objectToProxyFor, args);
+ try
+ {
+ return method.invoke(objectToProxyFor, args);
+ } catch (IllegalAccessException ex)
+ {
+ method.setAccessible(true);
+ return method.invoke(objectToProxyFor, args);
+ }
} catch (InvocationTargetException ex)
{
throw ex.getTargetException();
@@ -153,6 +193,64 @@ public class MonitoringProxy<T>
public Object invoke(final Object myProxy, final Method method, final Object[] args)
throws Throwable
+ {
+ final ExecutionResult<Object> result = retryingExecuteInThread(myProxy, method, args);
+ if (result.getStatus() == ExecutionStatus.TIMED_OUT)
+ {
+ if (errorValueOnTimeout == false)
+ {
+ throw new TimeoutException(describe(method) + " timed out (timeout="
+ + timingParameters.getTimeoutMillis() + "ms).");
+ }
+ return getErrorValue(method);
+ }
+ if (result.getStatus() == ExecutionStatus.INTERRUPTED && errorValueOnInterrupt)
+ {
+ return getErrorValue(method);
+ }
+ return ConcurrencyUtilities.tryDealWithResult(result);
+ }
+
+ private ExecutionResult<Object> retryingExecuteInThread(final Object myProxy,
+ final Method method, final Object[] args)
+ {
+ int counter = 0;
+ ExecutionResult<Object> result = null;
+ do
+ {
+ result = executeInThread(myProxy, method, args);
+ if (result.getStatus() == ExecutionStatus.COMPLETE
+ || result.getStatus() == ExecutionStatus.INTERRUPTED)
+ {
+ break;
+ }
+ if (exceptionStatusUnsuitableForRetry(result))
+ {
+ break;
+ }
+ if (counter > 0 && timingParameters.getIntervalToWaitAfterFailureMillis() > 0)
+ {
+ try
+ {
+ Thread.sleep(timingParameters.getIntervalToWaitAfterFailureMillis());
+ } catch (InterruptedException ex)
+ {
+ return ExecutionResult.createInterrupted();
+ }
+ }
+ } while (counter++ < timingParameters.getMaxRetriesOnFailure());
+ return result;
+ }
+
+ private boolean exceptionStatusUnsuitableForRetry(ExecutionResult<Object> result)
+ {
+ return (result.getStatus() == ExecutionStatus.EXCEPTION)
+ && exceptionClassesSuitableForRetrying.contains(result.tryGetException()
+ .getClass()) == false;
+ }
+
+ private ExecutionResult<Object> executeInThread(final Object myProxy, final Method method,
+ final Object[] args)
{
final String callingThreadName = Thread.currentThread().getName();
final Future<Object> future = executor.submit(new NamedCallable<Object>()
@@ -185,22 +283,32 @@ public class MonitoringProxy<T>
}
}
});
- final ExecutionResult<Object> result =
- ConcurrencyUtilities.getResult(future, timeoutMillis);
- if (result.getStatus() == ExecutionStatus.TIMED_OUT)
- {
- if (errorValueOnTimeout == false)
- {
- throw new TimeoutException(describe(method) + " timed out (timeout="
- + timeoutMillis + "ms).");
- }
- return getErrorValue(method);
- }
- if (result.getStatus() == ExecutionStatus.INTERRUPTED && errorValueOnInterrupt)
- {
- return getErrorValue(method);
- }
- return ConcurrencyUtilities.tryDealWithResult(result);
+ final ILogSettings logSettingsOrNull =
+ (loggerOrNull == null) ? null : new ILogSettings()
+ {
+ public LogLevel getLogLevelForError()
+ {
+ return LogLevel.ERROR;
+ }
+
+ public ISimpleLogger getLogger()
+ {
+ return loggerOrNull;
+ }
+
+ public String getOperationName()
+ {
+ if (nameOrNull != null)
+ {
+ return describe(method) + "[" + nameOrNull + "]";
+ } else
+ {
+ return describe(method);
+ }
+ }
+ };
+ return ConcurrencyUtilities.getResult(future, timingParameters.getTimeoutMillis(),
+ true, logSettingsOrNull, sensorOrNull);
}
private Object getErrorValue(final Method method)
@@ -223,7 +331,8 @@ public class MonitoringProxy<T>
this.errorTypeValueMap = createDefaultErrorTypeValueMap();
this.errorMethodValueMap = new HashMap<Method, Object>();
- this.timeoutMillis = ConcurrencyUtilities.NO_TIMEOUT;
+ this.exceptionClassesSuitableForRetrying = new HashSet<Class<? extends Exception>>();
+ this.timingParameters = TimingParameters.getNoTimeoutNoRetriesParameters();
this.delegate = new DelegatingInvocationHandler<T>(objectToProxyFor);
this.handler = new MonitoringInvocationHandler();
this.proxy = createProxy(interfaceClass, objectToProxyFor);
@@ -255,13 +364,14 @@ public class MonitoringProxy<T>
}
/**
- * Sets the timeout to <var>newTimeoutMillis</var>.
+ * Sets the timing parameters to <var>newParameters</var>.
*
* @return This object (for chaining).
*/
- public MonitoringProxy<T> timeoutMillis(long newTimeoutMillis)
+ public MonitoringProxy<T> timing(TimingParameters newParameters)
{
- this.timeoutMillis = newTimeoutMillis;
+ assert newParameters != null;
+ this.timingParameters = newParameters;
return this;
}
@@ -298,6 +408,17 @@ public class MonitoringProxy<T>
return this;
}
+ /**
+ * Sets the logger to be used for error logging to <var>newLogger</var>.
+ *
+ * @return This object (for chaining).
+ */
+ public MonitoringProxy<T> errorLog(ISimpleLogger newLogger)
+ {
+ this.loggerOrNull = newLogger;
+ return this;
+ }
+
/**
* Sets an error return <var>value</var> for the type <var>clazz</var>.
* <p>
@@ -323,6 +444,35 @@ public class MonitoringProxy<T>
return this;
}
+ /**
+ * Add an {@link Exception} class that is suitable for retrying the operation.
+ */
+ public MonitoringProxy<T> exceptionClassSuitableForRetrying(Class<? extends Exception> clazz)
+ {
+ exceptionClassesSuitableForRetrying.add(clazz);
+ return this;
+ }
+
+ /**
+ * Add a list of {@link Exception} classes that are suitable for retrying the operation.
+ */
+ public MonitoringProxy<T> exceptionClassesSuitableForRetrying(
+ Collection<Class<? extends Exception>> classes)
+ {
+ exceptionClassesSuitableForRetrying.addAll(classes);
+ return this;
+ }
+
+ /**
+ * Sets an sensor of fine-grained activity. Activity on this sensor can prevent a method
+ * invocation from timing out.
+ */
+ public MonitoringProxy<T> sensor(IActivitySensor sensor)
+ {
+ this.sensorOrNull = sensor;
+ return this;
+ }
+
/**
* Returns the actual proxy.
*/
diff --git a/common/source/java/ch/systemsx/cisd/common/concurrent/RecordingActivityObserverSensor.java b/common/source/java/ch/systemsx/cisd/common/concurrent/RecordingActivityObserverSensor.java
new file mode 100644
index 0000000000000000000000000000000000000000..3840aac57765c55673ab8679f7433bcac30fd478
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/concurrent/RecordingActivityObserverSensor.java
@@ -0,0 +1,62 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.concurrent;
+
+/**
+ * An implementation of {@link IActivityObserver} that just records the last event.
+ *
+ * @author Bernd Rinn
+ */
+public class RecordingActivityObserverSensor implements IActivityObserver, IActivitySensor
+{
+
+ private volatile long lastActivityMillis;
+
+ public RecordingActivityObserverSensor()
+ {
+ update(); // Avoid initial 'false alarm'.
+ }
+
+ protected long getLastActivityMillis()
+ {
+ return lastActivityMillis;
+ }
+
+ //
+ // IActivityObserver
+ //
+
+ public void update()
+ {
+ lastActivityMillis = System.currentTimeMillis();
+ }
+
+ //
+ // IActivitySensor
+ //
+
+ public long getLastActivityMillisMoreRecentThan(long thresholdMillis)
+ {
+ return lastActivityMillis;
+ }
+
+ public boolean hasActivityMoreRecentThan(long thresholdMillis)
+ {
+ return (System.currentTimeMillis() - lastActivityMillis) < thresholdMillis;
+ }
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/exceptions/InvalidExternalDataException.java b/common/source/java/ch/systemsx/cisd/common/exceptions/InvalidExternalDataException.java
new file mode 100644
index 0000000000000000000000000000000000000000..19f9e611497ee42e8ff1f57a8a6363571b4d5749
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/exceptions/InvalidExternalDataException.java
@@ -0,0 +1,57 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.exceptions;
+
+/**
+ * Data provided to the system from another system is invalid. This class is supposed to be used in
+ * cases where the data cannot be attributed to any user, but rather another system.
+ *
+ * @author Bernd Rinn
+ */
+public class InvalidExternalDataException extends HighLevelException
+{
+
+ private static final long serialVersionUID = 1L;
+
+ public InvalidExternalDataException(String message)
+ {
+ super(message);
+ }
+
+ public InvalidExternalDataException(String message, Throwable cause)
+ {
+ super(message, cause);
+ }
+
+ /**
+ * Creates an {@link InvalidExternalDataException} using a {@link java.util.Formatter}.
+ */
+ public static InvalidExternalDataException fromTemplate(String messageTemplate, Object... args)
+ {
+ return new InvalidExternalDataException(String.format(messageTemplate, args));
+ }
+
+ /**
+ * Creates an {@link InvalidExternalDataException} using a {@link java.util.Formatter}.
+ */
+ public static InvalidExternalDataException fromTemplate(Throwable cause, String messageTemplate,
+ Object... args)
+ {
+ return new InvalidExternalDataException(String.format(messageTemplate, args), cause);
+ }
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/FileLinkUtilities.java b/common/source/java/ch/systemsx/cisd/common/filesystem/FileLinkUtilities.java
index 3d9eae3c8994c34d379fb83592b780dea06dfa93..f23d6d7e7002e291d987eb950e9b5c4ba0c77c12 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/FileLinkUtilities.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/FileLinkUtilities.java
@@ -16,6 +16,9 @@
package ch.systemsx.cisd.common.filesystem;
+import java.io.IOException;
+
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
import ch.systemsx.cisd.common.utilities.FileUtilities;
/**
@@ -28,28 +31,17 @@ public class FileLinkUtilities
private final static boolean operational = FileUtilities.loadNativeLibraryFromResource("jlink");
- /** An exception that indicates that creating a link failed. */
- public static final class FileLinkException extends RuntimeException
+ private static void throwException(String type, String source, String target,
+ String errorMessage)
{
- private static final long serialVersionUID = 1L;
-
- private FileLinkException(String type, String source, String target, String errorMessage)
- {
- super(String.format("Creating %s link '%s' -> '%s': %s", type, source, target,
- errorMessage));
- }
+ throw new WrappedIOException(new IOException(String.format(
+ "Creating %s link '%s' -> '%s': %s", type, source, target, errorMessage)));
}
- /** An exception that indicates that obtaining information about a link failed. */
- public static final class FileLinkInfoException extends RuntimeException
+ private static void throwException(String filename, String errorMessage)
{
- private static final long serialVersionUID = 1L;
-
- private FileLinkInfoException(String filename, String errorMessage)
- {
- super(String.format("Cannot obtain inode info for file '%s': %s", filename,
- errorMessage));
- }
+ throw new WrappedIOException(new IOException(String.format(
+ "Cannot obtain inode info for file '%s': %s", filename, errorMessage)));
}
/** A class that provides information about a link. */
@@ -131,54 +123,55 @@ public class FileLinkUtilities
/**
* Creates a hard link from <var>filename</var> to <var>linkname</var>.
*
- * @throws FileLinkException If the underlying system call fails, e.g. because <var>filename</var>
- * does not exist or because <var>linkname</var> already exists.
+ * @throws WrappedIOException If the underlying system call fails, e.g. because
+ * <var>filename</var> does not exist or because <var>linkname</var> already exists.
*/
public static final void createHardLink(String filename, String linkname)
- throws FileLinkException
+ throws WrappedIOException
{
if (filename == null || linkname == null)
{
- throw new FileLinkException("hard", filename, linkname, "null is not allowed");
+ throwException("hard", filename, linkname, "null is not allowed");
}
final int result = hardlink(filename, linkname);
if (result < 0)
{
- throw new FileLinkException("hard", filename, linkname, strerror(result));
+ throwException("hard", filename, linkname, strerror(result));
}
}
/**
* Creates a symbolic link from <var>filename</var> to <var>linkname</var>.
*
- * @throws FileLinkException If the underlying system call fails, e.g. because <var>filename</var>
- * does not exist or because <var>linktarget</var> already exists.
+ * @throws WrappedIOException If the underlying system call fails, e.g. because
+ * <var>filename</var> does not exist or because <var>linktarget</var> already
+ * exists.
*/
public static final void createSymbolicLink(String filename, String linkname)
- throws FileLinkException
+ throws WrappedIOException
{
if (filename == null || linkname == null)
{
- throw new FileLinkException("symbolic", filename, linkname, "null is not allowed");
+ throwException("symbolic", filename, linkname, "null is not allowed");
}
final int result = symlink(filename, linkname);
if (result < 0)
{
- throw new FileLinkException("symbolic", filename, linkname, strerror(result));
+ throwException("symbolic", filename, linkname, strerror(result));
}
}
- private static int[] getLinkInfoArray(String linkname) throws FileLinkInfoException
+ private static int[] getLinkInfoArray(String linkname) throws WrappedIOException
{
if (linkname == null)
{
- throw new FileLinkInfoException(linkname, "null is not allowed");
+ throwException(linkname, "null is not allowed");
}
final int[] inodeInfo = new int[4];
final int result = linkinfo(linkname, inodeInfo);
if (result < 0)
{
- throw new FileLinkInfoException(linkname, strerror(result));
+ throwException(linkname, strerror(result));
}
return inodeInfo;
}
@@ -186,10 +179,10 @@ public class FileLinkUtilities
/**
* Returns the inode for the <var>filename</var>.
*
- * @throws FileLinkInfoException If the information could not be obtained, e.g. because the link
+ * @throws WrappedIOException If the information could not be obtained, e.g. because the link
* does not exist.
*/
- public static final int getInode(String filename) throws FileLinkInfoException
+ public static final int getInode(String filename) throws WrappedIOException
{
return getLinkInfoArray(filename)[0];
}
@@ -197,10 +190,10 @@ public class FileLinkUtilities
/**
* Returns the inode for the <var>filename</var>.
*
- * @throws FileLinkInfoException If the information could not be obtained, e.g. because the link
+ * @throws WrappedIOException If the information could not be obtained, e.g. because the link
* does not exist.
*/
- public static final int getHardLinkCount(String filename) throws FileLinkInfoException
+ public static final int getHardLinkCount(String filename) throws WrappedIOException
{
return getLinkInfoArray(filename)[1];
}
@@ -209,10 +202,10 @@ public class FileLinkUtilities
* Returns <code>true</code> if <var>filename</var> is a symbolic link and <code>false</code>
* otherwise.
*
- * @throws FileLinkInfoException If the information could not be obtained, e.g. because the link
+ * @throws WrappedIOException If the information could not be obtained, e.g. because the link
* does not exist.
*/
- public static final boolean isSymbolicLink(String filename) throws FileLinkInfoException
+ public static final boolean isSymbolicLink(String filename) throws WrappedIOException
{
return getLinkInfoArray(filename)[2] != 0;
}
@@ -221,10 +214,10 @@ public class FileLinkUtilities
* Returns the value of the symbolik link <var>filename</var>, or <code>null</code>, if
* <var>filename</var> is not a symbolic link.
*
- * @throws FileLinkInfoException If the information could not be obtained, e.g. because the link
+ * @throws WrappedIOException If the information could not be obtained, e.g. because the link
* does not exist.
*/
- public static final String tryReadSymbolicLink(String linkname) throws FileLinkInfoException
+ public static final String tryReadSymbolicLink(String linkname) throws WrappedIOException
{
final int[] info = getLinkInfoArray(linkname);
return (info[2] != 0) ? readlink(linkname, info[3]) : null;
@@ -233,10 +226,10 @@ public class FileLinkUtilities
/**
* Returns the information about <var>linkname</var>.
*
- * @throws FileLinkInfoException If the information could not be obtained, e.g. because the link
+ * @throws WrappedIOException If the information could not be obtained, e.g. because the link
* does not exist.
*/
- public static final LinkInfo getLinkInfo(String linkname) throws FileLinkInfoException
+ public static final LinkInfo getLinkInfo(String linkname) throws WrappedIOException
{
final int[] info = getLinkInfoArray(linkname);
final String symbolicLinkOrNull = (info[2] != 0) ? readlink(linkname, info[3]) : null;
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/FileOperations.java b/common/source/java/ch/systemsx/cisd/common/filesystem/FileOperations.java
new file mode 100644
index 0000000000000000000000000000000000000000..4a1b31da285d0c2f32316683adc28fc82eacb6a1
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/FileOperations.java
@@ -0,0 +1,497 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.List;
+
+import org.apache.commons.io.FileCopyUtils;
+import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.IOUtils;
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.concurrent.IActivityObserver;
+import ch.systemsx.cisd.common.concurrent.MonitoringProxy;
+import ch.systemsx.cisd.common.concurrent.RecordingActivityObserverSensor;
+import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
+import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.utilities.FileUtilities;
+
+/**
+ * Monitoring for (potentially hanging) file operations. Supposed to be used on remote file systems.
+ *
+ * @author Bernd Rinn
+ */
+public class FileOperations implements IFileOperations
+{
+
+ // @Private
+ final static Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, IFileOperations.class);
+
+ private final static IFileOperations instance = new FileOperations(null, null);
+
+ private final static ThreadLocal<IFileOperations> monitoredInstanceForThread =
+ new ThreadLocal<IFileOperations>();
+
+ /**
+ * Returns the straight-forward implementation of {@link IFileOperations}.
+ */
+ public static IFileOperations getInstance()
+ {
+ return instance;
+ }
+
+ /**
+ * Returns the monitored implementation of {@link IFileRemover} for the current thread with the
+ * default {@link TimingParameters} (i.e. with {@link TimingParameters#getDefaultParameters()}).
+ * <p>
+ * Note that you should only ever cache the return value of this method if you can ensure that
+ * the cached value will only be used in the same thread as the one where this method has been
+ * called. The monitored instance is <i>not</i> thread-safe with respect to the recursive list,
+ * remove and copy methods.
+ */
+ public static IFileOperations getMonitoredInstanceForCurrentThread()
+ {
+ IFileOperations monitoredInstance = monitoredInstanceForThread.get();
+ if (monitoredInstance == null)
+ {
+ monitoredInstance =
+ internalCreateMonitored(TimingParameters.getDefaultParameters(),
+ new RecordingActivityObserverSensor());
+ monitoredInstanceForThread.set(monitoredInstance);
+ }
+
+ return monitoredInstance;
+ }
+
+ /**
+ * Creates a monitored implementation of {@link IFileRemover} with the given
+ * <var>observerSensor</var>.
+ */
+ public static IFileOperations createMonitoredInstance(
+ RecordingActivityObserverSensor observerSensor)
+ {
+ return internalCreateMonitored(TimingParameters.getDefaultParameters(), observerSensor);
+ }
+
+ /**
+ * Creates a monitored implementation of {@link IFileRemover} with the given
+ * <var>parameters</var>.
+ */
+ public static IFileOperations createMonitoredInstance(TimingParameters parameters)
+ {
+ // One of the rare cases where '==' is exactly what we mean. Note that TimingParameters are
+ // <i>not</i> immutable.
+ if (parameters == TimingParameters.getDefaultParameters())
+ {
+ return getMonitoredInstanceForCurrentThread();
+ } else
+ {
+ return internalCreateMonitored(parameters, new RecordingActivityObserverSensor());
+ }
+ }
+
+ private static IFileOperations internalCreateMonitored(TimingParameters parameters,
+ RecordingActivityObserverSensor observerSensor)
+ {
+ return MonitoringProxy.create(IFileOperations.class,
+ new FileOperations(parameters, observerSensor)).timing(parameters).sensor(
+ observerSensor).errorLog(new Log4jSimpleLogger(operationLog)).name(
+ "remote file operations").get();
+ }
+
+ private final TimingParameters timingParametersOrNull;
+
+ private final IActivityObserver observerOrNull;
+
+ // @Private
+ FileOperations(TimingParameters timingParametersOrNull, IActivityObserver observerOrNull)
+ {
+ this.timingParametersOrNull = timingParametersOrNull;
+ this.observerOrNull = observerOrNull;
+ }
+
+ //
+ // IFileOperations
+ //
+
+ // java.io.File
+
+ public boolean exists(File file)
+ {
+ return file.exists();
+ }
+
+ public boolean delete(File file)
+ {
+ return file.delete();
+ }
+
+ public boolean rename(File source, File destination)
+ {
+ return source.renameTo(destination);
+ }
+
+ public boolean canRead(File file)
+ {
+ return file.canRead();
+ }
+
+ public boolean canWrite(File file)
+ {
+ return file.canWrite();
+ }
+
+ public boolean createNewFile(File file) throws WrappedIOException
+ {
+ try
+ {
+ return file.createNewFile();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public File createTempFile(String prefix, String suffix, File directory)
+ throws WrappedIOException
+ {
+ try
+ {
+ return File.createTempFile(prefix, suffix, directory);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public File createTempFile(String prefix, String suffix) throws WrappedIOException
+ {
+ try
+ {
+ return File.createTempFile(prefix, suffix);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public File getCanonicalFile(File file) throws WrappedIOException
+ {
+ try
+ {
+ return file.getCanonicalFile();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public String getCanonicalPath(File file) throws WrappedIOException
+ {
+ try
+ {
+ return file.getCanonicalPath();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public boolean isDirectory(File file)
+ {
+ return file.isDirectory();
+ }
+
+ public boolean isFile(File file)
+ {
+ return file.isFile();
+ }
+
+ public boolean isHidden(File file)
+ {
+ return file.isHidden();
+ }
+
+ public long lastModified(File file)
+ {
+ return file.lastModified();
+ }
+
+ public long length(File file)
+ {
+ return file.length();
+ }
+
+ public String[] list(File file)
+ {
+ return file.list();
+ }
+
+ public String[] list(File file, FilenameFilter filter)
+ {
+ return file.list(filter);
+ }
+
+ public File[] listFiles(File file)
+ {
+ return file.listFiles();
+ }
+
+ public File[] listFiles(File file, FilenameFilter filter)
+ {
+ return file.listFiles(filter);
+ }
+
+ public File[] listFiles(File file, FileFilter filter)
+ {
+ return file.listFiles(filter);
+ }
+
+ public boolean mkdir(File file)
+ {
+ return file.mkdir();
+ }
+
+ public boolean mkdirs(File file)
+ {
+ return file.mkdirs();
+ }
+
+ public boolean setLastModified(File file, long time)
+ {
+ return file.setLastModified(time);
+ }
+
+ public boolean setReadOnly(File file)
+ {
+ return file.setReadOnly();
+ }
+
+ // Advanced
+
+ public List<File> listDirectories(File directory, boolean recursive)
+ {
+ return FileUtilities.listDirectories(directory, recursive, observerOrNull);
+ }
+
+ public List<File> listFiles(File directory, String[] extensionsOrNull, boolean recursive)
+ {
+ return FileUtilities.listFiles(directory, extensionsOrNull, recursive, observerOrNull);
+ }
+
+ public List<File> listFilesAndDirectories(File directory, boolean recursive)
+ {
+ return FileUtilities.listFilesAndDirectories(directory, recursive, observerOrNull);
+ }
+
+ public void touch(File file) throws WrappedIOException
+ {
+ try
+ {
+ FileUtils.touch(file);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void deleteRecursively(File fileToRemove) throws WrappedIOException
+ {
+ final boolean deleteOK =
+ FileUtilities.deleteRecursively(fileToRemove, null, observerOrNull);
+ if (deleteOK == false)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(new IOException("Recursive deletion of '"
+ + fileToRemove + "' failed."));
+ }
+ }
+
+ public boolean removeRecursively(File fileToRemove)
+ {
+ return FileUtilities.deleteRecursively(fileToRemove, null, observerOrNull);
+ }
+
+ public void move(File source, File destination) throws WrappedIOException
+ {
+ if (destination.isDirectory())
+ {
+ moveToDirectory(source, destination);
+ } else
+ {
+ final boolean renameOK = rename(source, destination);
+ if (renameOK == false)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(new IOException("Moving '"
+ + source.getAbsolutePath() + "' into directory '"
+ + destination.getAbsolutePath() + "' failed."));
+ }
+ }
+ }
+
+ public void moveToDirectory(File source, File destinationDirectory) throws WrappedIOException
+ {
+ final File target = new File(destinationDirectory, source.getName());
+ final boolean moveOK = source.renameTo(target);
+ if (moveOK == false)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(new IOException("Moving '"
+ + source.getAbsolutePath() + "' into directory '"
+ + destinationDirectory.getAbsolutePath() + "' failed."));
+ }
+ }
+
+ public void copyDirectory(File srcDir, File destDir)
+ {
+ try
+ {
+ FileCopyUtils.copyDirectory(srcDir, destDir, observerOrNull);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void copyDirectoryToDirectory(File srcDir, File destDir) throws WrappedIOException
+ {
+ try
+ {
+ FileCopyUtils.copyDirectoryToDirectory(srcDir, destDir, observerOrNull);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void copyFile(File srcFile, File destFile) throws WrappedIOException
+ {
+ try
+ {
+ FileCopyUtils.copyFile(srcFile, destFile);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void copyFileToDirectory(File srcFile, File destDir) throws WrappedIOException
+ {
+ try
+ {
+ FileCopyUtils.copyFileToDirectory(srcFile, destDir);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void copy(File source, File destination) throws WrappedIOException
+ {
+ try
+ {
+ FileCopyUtils.copy(source, destination);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void copyToDirectory(File source, File destDir) throws WrappedIOException
+ {
+ try
+ {
+ FileCopyUtils.copyToDirectory(source, destDir);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public byte[] getContentAsByteArray(File file) throws WrappedIOException
+ {
+ java.io.InputStream inputStream = null;
+ try
+ {
+ inputStream = new FileInputStream(file);
+ return IOUtils.toByteArray(inputStream);
+ } catch (IOException ex)
+ {
+ throw new WrappedIOException(ex);
+ } finally
+ {
+ IOUtils.closeQuietly(inputStream);
+ }
+ }
+
+ public String getContentAsString(File file) throws WrappedIOException
+ {
+ return FileUtilities.loadToString(file);
+ }
+
+ public List<String> getContentAsStringList(File file) throws WrappedIOException
+ {
+ return FileUtilities.loadToStringList(file);
+ }
+
+ public InputStream getInputStream(File file) throws WrappedIOException
+ {
+ return new IInputStreamAdapter(getIInputStream(file));
+ }
+
+ public IInputStream getIInputStream(File file) throws WrappedIOException
+ {
+ try
+ {
+ final IInputStream is = internalGetIInputStream(file);
+ if (timingParametersOrNull != null)
+ {
+ return MonitoringProxy.create(IInputStream.class, is).timing(
+ timingParametersOrNull).name(
+ "input stream <" + file.getAbsolutePath() + ">").get();
+ } else
+ {
+ return is;
+ }
+ } catch (IOException ex)
+ {
+ throw new WrappedIOException(ex);
+ }
+ }
+
+ // @Private
+ IInputStream internalGetIInputStream(File file) throws FileNotFoundException
+ {
+ return new InputStreamAdapter(new FileInputStream(file));
+ }
+
+ public void writeToFile(File file, String content) throws WrappedIOException
+ {
+ FileUtilities.writeToFile(file, content);
+ }
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/IFileOperations.java b/common/source/java/ch/systemsx/cisd/common/filesystem/IFileOperations.java
new file mode 100644
index 0000000000000000000000000000000000000000..717e6cca23e2fa0f33fc3b8a67c1bdfde759b301
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/IFileOperations.java
@@ -0,0 +1,379 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FilenameFilter;
+import java.io.InputStream;
+import java.util.List;
+
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
+
+/**
+ * Operations on {@link File} that are good to monitor when being performed on remote file systems
+ * because they can hang when a remote resource goes down unexpectedly.
+ *
+ * @author Bernd Rinn
+ */
+public interface IFileOperations extends IFileRemover
+{
+
+ //
+ // java.io.File
+ //
+
+ /**
+ * @see File#getCanonicalPath()
+ */
+ public String getCanonicalPath(File file) throws WrappedIOException;
+
+ /**
+ * @see File#getCanonicalFile()
+ */
+ public File getCanonicalFile(File file) throws WrappedIOException;
+
+ /**
+ * @see File#canRead()
+ */
+ public boolean canRead(File file);
+
+ /**
+ * @see File#canWrite()
+ */
+ public boolean canWrite(File file);
+
+ /**
+ * @see File#exists()
+ */
+ public boolean exists(File file);
+
+ /**
+ * @see File#isDirectory()
+ */
+ public boolean isDirectory(File file);
+
+ /**
+ * @see File#isFile()
+ */
+ public boolean isFile(File file);
+
+ /**
+ * @see File#isHidden()
+ */
+ public boolean isHidden(File file);
+
+ /**
+ * @see File#lastModified()
+ */
+ public long lastModified(File file);
+
+ /**
+ * @see File#length()
+ */
+ public long length(File file);
+
+ public boolean createNewFile(File file) throws WrappedIOException;
+
+ /**
+ * @see File#delete()
+ */
+ public boolean delete(File file);
+
+ /**
+ * @see File#list()
+ */
+ public String[] list(File file);
+
+ /**
+ * @see File#list(FilenameFilter)
+ */
+ public String[] list(File file, FilenameFilter filter);
+
+ /**
+ * @see File#listFiles()
+ */
+ public File[] listFiles(File file);
+
+ /**
+ * @see File#listFiles(FilenameFilter)
+ */
+ public File[] listFiles(File file, FilenameFilter filter);
+
+ /**
+ * @see File#listFiles(FileFilter)
+ */
+ public File[] listFiles(File file, FileFilter filter);
+
+ /**
+ * @see File#mkdir()
+ */
+ public boolean mkdir(File file);
+
+ /**
+ * @see File#mkdirs()
+ */
+ public boolean mkdirs(File file);
+
+ /**
+ * @see File#renameTo(File)
+ */
+ public boolean rename(File source, File destination);
+
+ public boolean setLastModified(File file, long time);
+
+ public boolean setReadOnly(File file);
+
+ public File createTempFile(String prefix, String suffix, File directory)
+ throws WrappedIOException;
+
+ public File createTempFile(String prefix, String suffix) throws WrappedIOException;
+
+ //
+ // Advanced
+ //
+
+ /**
+ * Finds files within a given directory (and optionally its subdirectories) which match an array
+ * of extensions.
+ * <p>
+ * This call is suitable for large or huge directories on slow file systems as it is able to
+ * notify the monitor of progress.
+ *
+ * @param directory The directory to search in.
+ * @param extensionsOrNull An array of extensions, ex. {"java","xml"}. If this parameter is
+ * <code>null</code>, all files are returned.
+ * @param recursive If true all subdirectories are searched as well.
+ * @return A list of java.io.File (all files) with the matching files, or an empty list, if
+ * <var>directory</var> ist not a directory.
+ */
+ public List<File> listFiles(File directory, String[] extensionsOrNull, boolean recursive);
+
+ /**
+ * Finds directories within a given directory (and optionally its subdirectories).
+ * <p>
+ * This call is suitable for large or huge directories on slow file systems as it is able to
+ * notify the monitor of progress.
+ *
+ * @param directory The directory to search in.
+ * @param recursive If true all subdirectories are searched as well.
+ * @return A list of java.io.File (all directories), or an empty list, if <var>directory</var>
+ * ist not a directory.
+ */
+ public List<File> listDirectories(File directory, boolean recursive);
+
+ /**
+ * Finds files or directories within a given directory (and optionally its subdirectories).
+ * <p>
+ * This call is suitable for large or huge directories on slow file systems as it is able to
+ * notify the monitor of progress.
+ *
+ * @param directory The directory to search in.
+ * @param recursive If true all subdirectories are searched as well.
+ * @return A list of java.io.File (all directories), or an empty list, if <var>directory</var>
+ * ist not a directory.
+ */
+ public List<File> listFilesAndDirectories(File directory, boolean recursive);
+
+ /**
+ * Sets the file's last modification time to the current time without changing the content. If
+ * the file does not exist, create it empty.
+ *
+ * @throws WrappedIOException If the file cannot be created or the last modification time cannot
+ * be changed.
+ */
+ public void touch(File file) throws WrappedIOException;
+
+ /**
+ * Removes the given <var>fileToRemove</var>, if necessary recursively.
+ *
+ * @param fileToRemove File or directory to remove. If it is a directory, it will be removed
+ * recursively.
+ * @throws WrappedIOException If the file or directory cannot be removed.
+ */
+ public void deleteRecursively(File fileToRemove) throws WrappedIOException;
+
+ /**
+ * Move <var>source</var> to <var>destinationDirectory</var>.
+ *
+ * @param source File or directory to move. Must exist when this method is called.
+ * @param destinationDirectory The directory to move <var>source</var> to. Has to be an existing
+ * directory.
+ * @throws WrappedIOException If <var>source</var> cannot be moved into
+ * <var>destinationDirectory</var>.
+ */
+ public void moveToDirectory(File source, File destinationDirectory) throws WrappedIOException;
+
+ /**
+ * Rename <var>source</var> to <var>destination</var>, or move <var>source</var> to
+ * <var>destination</var> if it is an existing directory. Combines {@link #rename(File, File)}
+ * and {@link #moveToDirectory(File, File)} in move method, choosing the right method depending
+ * on the <var>destination</var>.
+ *
+ * @param source File or directory to move. Must exist when this method is called.
+ * @param destination If it does not exist, this is the new name for <var>source</var>. If it is
+ * an existing directory, <var>source</var> will be moved to this directory.
+ * @throws WrappedIOException If <var>source</var> cannot be moved to <var>destination</var>.
+ */
+ public void move(File source, File destination) throws WrappedIOException;
+
+ /**
+ * Copies a file or a whole directory to a new location preserving the file dates.
+ * <p>
+ * Calls {@link #copyFile(File, File)} if <var>source</var> is a file and
+ * {@link #copyDirectory(File, File)} if it is a directory.
+ *
+ * @param source an existing file or directory to copy, must not be <code>null</code>
+ * @param destination the new file or directory, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws WrappedIOException if source or destination is invalid
+ * @throws WrappedIOException if an IO error occurs during copying
+ * @see #copyFile(File, File)
+ * @see #copyDirectory(File, File)
+ */
+ public void copy(File source, File destination) throws WrappedIOException;
+
+ /**
+ * Copies a file or a whole directory to a new location preserving the file dates.
+ * <p>
+ * Calls {@link #copyFileToDirectory(File, File)} if <var>source</var> is a file and
+ * {@link #copyDirectoryToDirectory(File, File)} if it is a directory.
+ *
+ * @param source an existing file or directory to copy, must not be <code>null</code>
+ * @param destDir the directory to place the copy in, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws WrappedIOException if source or destination is invalid
+ * @throws WrappedIOException if an IO error occurs during copying
+ * @see #copyFileToDirectory(File, File)
+ * @see #copyDirectoryToDirectory(File, File)
+ */
+ public void copyToDirectory(File source, File destDir) throws WrappedIOException;
+
+ /**
+ * Copies a whole directory to a new location preserving the file dates.
+ * <p>
+ * This method copies the specified directory and all its child directories and files to the
+ * specified destination. The destination is the new location and name of the directory.
+ * <p>
+ * The destination directory is created if it does not exist. If the destination directory did
+ * exist, then this method merges the source with the destination, with the source taking
+ * precedence.
+ *
+ * @param srcDir an existing directory to copy, must not be <code>null</code>
+ * @param destDir the new directory, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws WrappedIOException if source or destination is invalid
+ * @throws WrappedIOException if an IO error occurs during copying
+ */
+ public void copyDirectory(File srcDir, File destDir) throws WrappedIOException;
+
+ /**
+ * Copies a file to a new location preserving the file date.
+ * <p>
+ * This method copies the contents of the specified source file to the specified destination
+ * file. The directory holding the destination file is created if it does not exist. If the
+ * destination file exists, then this method will overwrite it.
+ *
+ * @param srcFile an existing file to copy, must not be <code>null</code>
+ * @param destFile the new file, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws WrappedIOException if source or destination is invalid
+ * @throws WrappedIOException if an IO error occurs during copying
+ * @see #copyFileToDirectory(File, File)
+ */
+ public void copyFile(File srcFile, File destFile) throws WrappedIOException;
+
+ /**
+ * Copies a file to a directory preserving the file date.
+ * <p>
+ * This method copies the contents of the specified source file to a file of the same name in
+ * the specified destination directory. The destination directory is created if it does not
+ * exist. If the destination file exists, then this method will overwrite it.
+ *
+ * @param srcFile an existing file to copy, must not be <code>null</code>
+ * @param destDir the directory to place the copy in, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is null
+ * @throws WrappedIOException if source or destination is invalid
+ * @throws WrappedIOException if an IO error occurs during copying
+ */
+ public void copyFileToDirectory(File srcFile, File destDir) throws WrappedIOException;
+
+ /**
+ * Copies a directory to within another directory preserving the file dates.
+ * <p>
+ * This method copies the source directory and all its contents to a directory of the same name
+ * in the specified destination directory.
+ * <p>
+ * The destination directory is created if it does not exist. If the destination directory did
+ * exist, then this method merges the source with the destination, with the source taking
+ * precedence.
+ *
+ * @param srcDir an existing directory to copy, must not be <code>null</code>
+ * @param destDir the directory to place the copy in, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws WrappedIOException if source or destination is invalid
+ * @throws WrappedIOException if an IO error occurs during copying
+ */
+ public void copyDirectoryToDirectory(File srcDir, File destDir) throws WrappedIOException;
+
+ //
+ // Content
+ //
+
+ /**
+ * Returns the content of <var>file</var> as a byte array.
+ *
+ * @throws WrappedIOException if an IO error occurs during reading the file
+ */
+ public byte[] getContentAsByteArray(File file) throws WrappedIOException;
+
+ /**
+ * Returns the content of <var>file</var> as a String.
+ *
+ * @throws WrappedIOException if an IO error occurs during reading the file
+ */
+ public String getContentAsString(File file) throws WrappedIOException;
+
+ /**
+ * Returns the content of <var>file</var> as a list of Strings.
+ *
+ * @throws WrappedIOException if an IO error occurs during reading the file
+ */
+ public List<String> getContentAsStringList(File file) throws WrappedIOException;
+
+ /**
+ * Returns a monitored {@link InputStream} of <var>file</var>.
+ *
+ * @throws WrappedIOException if an IO error occurs during opening the stream
+ */
+ public InputStream getInputStream(File file) throws WrappedIOException;
+
+ /**
+ * Returns a monitored {@link IInputStream} of <var>file</var>.
+ *
+ * @throws WrappedIOException if an IO error occurs during opening the stream
+ */
+ public IInputStream getIInputStream(File file) throws WrappedIOException;
+
+ /**
+ * Writes <var>content</var> to <var>file</var>. If <var>file</var> already exists, it will be
+ * overwritten.
+ *
+ * @throws WrappedIOException if an IO error occurs during writing
+ */
+ public void writeToFile(File file, String content) throws WrappedIOException;
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/IFileRemover.java b/common/source/java/ch/systemsx/cisd/common/filesystem/IFileRemover.java
new file mode 100644
index 0000000000000000000000000000000000000000..529a4b76b5afa9255ef0ea81c320bc1e96608fa2
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/IFileRemover.java
@@ -0,0 +1,37 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.File;
+
+/**
+ * A role that can remove a file or directory.
+ *
+ * @author Bernd Rinn
+ */
+public interface IFileRemover
+{
+ /**
+ * Removes the given <var>fileToRemove</var>.
+ *
+ * @param fileToRemove File or directory to remove. If it is a directory, it will be removed
+ * recursively.
+ * @return <code>true</code> if the file or directory was removed successfully and
+ * <code>false</code> otherwise.
+ */
+ public boolean removeRecursively(File fileToRemove);
+}
\ No newline at end of file
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/IInputStream.java b/common/source/java/ch/systemsx/cisd/common/filesystem/IInputStream.java
new file mode 100644
index 0000000000000000000000000000000000000000..d1985d6601330a2a895c0bdc78df3ead22b07425
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/IInputStream.java
@@ -0,0 +1,74 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
+
+/**
+ * In interface for {@link java.io.InputStream}.
+ *
+ * @author Bernd Rinn
+ */
+public interface IInputStream
+{
+
+ /**
+ * @see java.io.InputStream#read()
+ */
+ public int read() throws WrappedIOException;
+
+ /**
+ * @see java.io.InputStream#read(byte[])
+ */
+ public int read(byte b[]) throws WrappedIOException;
+
+ /**
+ * @see java.io.InputStream#read(byte[], int, int)
+ */
+ public int read(byte b[], int off, int len) throws WrappedIOException;
+
+ /**
+ * @see java.io.InputStream#skip(long)
+ */
+ public long skip(long n) throws WrappedIOException;
+
+ /**
+ * @see java.io.InputStream#available()
+ */
+ public int available() throws WrappedIOException;
+
+ /**
+ * @see java.io.InputStream#close()
+ */
+ public void close() throws WrappedIOException;
+
+ /**
+ * @see java.io.InputStream#mark(int)
+ */
+ public void mark(int readlimit);
+
+ /**
+ * @see java.io.InputStream#reset()
+ */
+ public void reset() throws WrappedIOException;
+
+ /**
+ * @see java.io.InputStream#markSupported()
+ */
+ public boolean markSupported();
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/IInputStreamAdapter.java b/common/source/java/ch/systemsx/cisd/common/filesystem/IInputStreamAdapter.java
new file mode 100644
index 0000000000000000000000000000000000000000..d4cab963c7bdffd50ca1d8eaf62255236782e3c3
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/IInputStreamAdapter.java
@@ -0,0 +1,135 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.IOException;
+import java.io.InputStream;
+
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
+
+/**
+ * An adapter for {@link IInputStream} that extends {@link java.io.InputStream}.
+ *
+ * @author Bernd Rinn
+ */
+public class IInputStreamAdapter extends InputStream
+{
+
+ private final IInputStream delegate;
+
+ public IInputStreamAdapter(IInputStream delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public int available() throws IOException
+ {
+ try
+ {
+ return delegate.available();
+ } catch (WrappedIOException ex)
+ {
+ throw (IOException) ex.getCause();
+ }
+ }
+
+ @Override
+ public void close() throws IOException
+ {
+ try
+ {
+ delegate.close();
+ } catch (WrappedIOException ex)
+ {
+ throw (IOException) ex.getCause();
+ }
+ }
+
+ @Override
+ public synchronized void mark(int readlimit)
+ {
+ delegate.mark(readlimit);
+ }
+
+ @Override
+ public boolean markSupported()
+ {
+ return delegate.markSupported();
+ }
+
+ @Override
+ public int read() throws IOException
+ {
+ try
+ {
+ return delegate.read();
+ } catch (WrappedIOException ex)
+ {
+ throw (IOException) ex.getCause();
+ }
+ }
+
+ @Override
+ public int read(byte[] b, int off, int len) throws IOException
+ {
+ try
+ {
+ return delegate.read(b, off, len);
+ } catch (WrappedIOException ex)
+ {
+ throw (IOException) ex.getCause();
+ }
+ }
+
+ @Override
+ public int read(byte[] b) throws IOException
+ {
+ try
+ {
+ return delegate.read(b);
+ } catch (WrappedIOException ex)
+ {
+ throw (IOException) ex.getCause();
+ }
+ }
+
+ @Override
+ public synchronized void reset() throws IOException
+ {
+ try
+ {
+ delegate.reset();
+ } catch (WrappedIOException ex)
+ {
+ throw (IOException) ex.getCause();
+ }
+ }
+
+ @Override
+ public long skip(long n) throws IOException
+ {
+ try
+ {
+ return delegate.skip(n);
+ } catch (WrappedIOException ex)
+ {
+ throw (IOException) ex.getCause();
+ }
+ }
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/InputStreamAdapter.java b/common/source/java/ch/systemsx/cisd/common/filesystem/InputStreamAdapter.java
new file mode 100644
index 0000000000000000000000000000000000000000..15e6d36fb2fff09049d7debeab295dcfa2b3aec5
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/InputStreamAdapter.java
@@ -0,0 +1,148 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.IOException;
+
+import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
+
+/**
+ * An adapter for {@link java.io.InputStream} that implements {@link IInputStream}.
+ *
+ * @author Bernd Rinn
+ */
+public class InputStreamAdapter implements IInputStream
+{
+
+ private final java.io.InputStream delegate;
+
+ public InputStreamAdapter(java.io.InputStream delegate)
+ {
+ this.delegate = delegate;
+ }
+
+ public int available() throws WrappedIOException
+ {
+ try
+ {
+ return delegate.available();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void close() throws WrappedIOException
+ {
+ try
+ {
+ delegate.close();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void mark(int readlimit)
+ {
+ delegate.mark(readlimit);
+ }
+
+ public boolean markSupported()
+ {
+ return delegate.markSupported();
+ }
+
+ public int read() throws WrappedIOException
+ {
+ try
+ {
+ return delegate.read();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public int read(byte[] b, int off, int len) throws WrappedIOException
+ {
+ try
+ {
+ return delegate.read(b, off, len);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public int read(byte[] b) throws WrappedIOException
+ {
+ try
+ {
+ return delegate.read(b);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public void reset() throws WrappedIOException
+ {
+ try
+ {
+ delegate.reset();
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ public long skip(long n) throws WrappedIOException
+ {
+ try
+ {
+ return delegate.skip(n);
+ } catch (IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
+ }
+ }
+
+ //
+ // Object
+ //
+
+ @Override
+ public String toString()
+ {
+ return delegate.toString();
+ }
+
+ @Override
+ public boolean equals(Object obj)
+ {
+ return delegate.equals(obj);
+ }
+
+ @Override
+ public int hashCode()
+ {
+ return delegate.hashCode();
+ }
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/LoggingPathRemoverDecorator.java b/common/source/java/ch/systemsx/cisd/common/filesystem/LoggingPathRemoverDecorator.java
new file mode 100644
index 0000000000000000000000000000000000000000..f4cf734e0516eb0d8abcd4d91cd733e23bc40bb9
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/LoggingPathRemoverDecorator.java
@@ -0,0 +1,66 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.File;
+
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.logging.LogLevel;
+
+/**
+ * An {@link IFileRemover} decorator that logs path removals.
+ *
+ * @author Bernd Rinn
+ */
+public class LoggingPathRemoverDecorator implements IFileRemover
+{
+ private final IFileRemover delegate;
+
+ private final ISimpleLogger logger;
+
+ private final boolean failuresOnly;
+
+ public LoggingPathRemoverDecorator(IFileRemover delegate, ISimpleLogger logger,
+ boolean failuresOnly)
+ {
+ this.delegate = delegate;
+ this.logger = logger;
+ this.failuresOnly = failuresOnly;
+ }
+
+ public boolean removeRecursively(File fileToRemove)
+ {
+ final boolean ok = delegate.removeRecursively(fileToRemove);
+ if (shouldLog(ok))
+ {
+ logger.log(ok ? LogLevel.INFO : LogLevel.ERROR, String.format("Deleting %s '%s': %s.",
+ getType(fileToRemove), fileToRemove.getPath(), ok ? "OK" : "FAILED"));
+ }
+ return ok;
+ }
+
+ private boolean shouldLog(final boolean ok)
+ {
+ return (ok && failuresOnly) == false;
+ }
+
+ private String getType(File fileToRemove)
+ {
+ return fileToRemove.isDirectory() ? "directory" : "file";
+ }
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/QueueingPathRemoverService.java b/common/source/java/ch/systemsx/cisd/common/filesystem/QueueingPathRemoverService.java
new file mode 100644
index 0000000000000000000000000000000000000000..d6626ce9aa0f09ede234b74912954497d09d0562
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/QueueingPathRemoverService.java
@@ -0,0 +1,229 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.collections.ExtendedBlockingQueueFactory;
+import ch.systemsx.cisd.common.collections.ExtendedLinkedBlockingQueue;
+import ch.systemsx.cisd.common.collections.IExtendedBlockingQueue;
+import ch.systemsx.cisd.common.collections.PersistentExtendedBlockingQueueDecorator;
+import ch.systemsx.cisd.common.collections.RecordBasedQueuePersister;
+import ch.systemsx.cisd.common.concurrent.MonitoringProxy;
+import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
+import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+
+/**
+ * A service for removing (deep) paths. It provides a {@link IFileRemover} that marks {@link File}s
+ * for deletion and queues them up, using a separate thread to actually delete them.
+ * <p>
+ * Note that the service needs to be started via {@link #start(File, TimingParameters)}.
+ * <p>
+ * A file can be specified that keeps track of all the items that are to be deleted in order to
+ * persist program restart.
+ *
+ * @author Bernd Rinn
+ */
+public class QueueingPathRemoverService implements IFileRemover
+{
+
+ private final static Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, QueueingPathRemoverService.class);
+
+ private static final int INITIAL_RECORD_SIZE = 128;
+
+ // @Private
+ final static String SHREDDER_PREFIX = ".SHREDDER_";
+
+ private static final AtomicInteger counter = new AtomicInteger();
+
+ private static IExtendedBlockingQueue<File> queue = null;
+
+ private static ICloseable queueCloseableOrNull = null;
+
+ private static Thread thread = null;
+
+ private static IFileRemover deepRemover = null;
+
+ private static IFileRemover queueingRemover = null;
+
+ private static QueueingPathRemoverService instance = new QueueingPathRemoverService();
+
+ /**
+ * Returns the instance of the shredder.
+ */
+ public static final QueueingPathRemoverService getInstance()
+ {
+ return instance;
+ }
+
+ /**
+ * Initializes the shredder thread. Will not persist over program restart. <i>Needs to be called
+ * before this class is constructed for the first time.</i>
+ */
+ public static final void start()
+ {
+ start(null);
+ }
+
+ /**
+ * Initializes the shredder thread. <i>Needs to be called before this class is constructed for
+ * the first time.</i>
+ *
+ * @param queueFileOrNull If not <code>null</code>, the file will be used to persist the items
+ * to be deleted over program restart.
+ */
+ public static final void start(File queueFileOrNull)
+ {
+ start(queueFileOrNull, TimingParameters.getDefaultParameters());
+ }
+
+ /**
+ * Initializes the shredder thread. <i>Needs to be called before this class is constructed for
+ * the first time.</i>
+ *
+ * @param queueFileOrNull If not <code>null</code>, the file will be used to persist the items
+ * to be deleted over program restart.
+ */
+ public static final void start(final File queueFileOrNull, TimingParameters parameters)
+ {
+ final ISimpleLogger logger = new Log4jSimpleLogger(operationLog);
+ final IFileRemover monitoringProxy = FileOperations.createMonitoredInstance(parameters);
+ deepRemover = new LoggingPathRemoverDecorator(monitoringProxy, logger, false);
+ queueingRemover = MonitoringProxy.create(IFileRemover.class, new IFileRemover()
+ {
+ public boolean removeRecursively(File fileToRemove)
+ {
+ if (fileToRemove.isFile())
+ {
+ return fileToRemove.delete();
+ } else
+ {
+ final String name =
+ SHREDDER_PREFIX + System.currentTimeMillis() + "-"
+ + counter.incrementAndGet() + "-" + fileToRemove.getName();
+ final File shredderFile = new File(fileToRemove.getParentFile(), name);
+ final boolean ok = fileToRemove.renameTo(shredderFile);
+ if (ok)
+ {
+ queue.add(shredderFile);
+ }
+ return ok;
+ }
+ }
+ }).timing(parameters).errorLog(logger).get();
+ if (queueFileOrNull != null)
+ {
+ final PersistentExtendedBlockingQueueDecorator<File> persistentQueue =
+ ExtendedBlockingQueueFactory.createPersistRecordBased(queueFileOrNull,
+ INITIAL_RECORD_SIZE);
+ queue = persistentQueue;
+ queueCloseableOrNull = persistentQueue;
+ } else
+ {
+ queue = new ExtendedLinkedBlockingQueue<File>();
+ }
+ thread = new Thread(new Runnable()
+ {
+ public void run()
+ {
+ try
+ {
+ while (true)
+ {
+ final File fileToRemove = queue.peekWait();
+ deepRemover.removeRecursively(fileToRemove);
+ // Note: this is the only consumer of this queue.
+ queue.take();
+ }
+ } catch (InterruptedException ex)
+ {
+ // Exit thread.
+ } catch (StopException ex)
+ {
+ // Exit thread.
+ }
+ }
+ }, "Shredder Queue");
+ thread.setDaemon(true);
+ thread.start();
+ }
+
+ private static final void close()
+ {
+ if (queueCloseableOrNull != null)
+ {
+ queueCloseableOrNull.close();
+ }
+ }
+
+ public static final void stop()
+ {
+ thread.interrupt();
+ close();
+ thread = null;
+ queue = null;
+ queueCloseableOrNull = null;
+ deepRemover = null;
+ }
+
+ public static final boolean stopAndWait(long timeoutMillis)
+ {
+ thread.interrupt();
+ try
+ {
+ thread.join(timeoutMillis);
+ } catch (InterruptedException ex)
+ {
+ }
+ final boolean ok = (thread.isAlive() == false);
+ close();
+ return ok;
+ }
+
+ public static final boolean isRunning()
+ {
+ return queueingRemover != null;
+ }
+
+ /**
+ * Return list of shredder items.
+ */
+ public static final List<File> listShredderItems(File queueFile)
+ {
+ return RecordBasedQueuePersister.list(File.class, queueFile);
+ }
+
+ private QueueingPathRemoverService()
+ {
+ // Cannot be instantiated.
+ }
+
+ public boolean removeRecursively(File fileToRemove)
+ {
+ return queueingRemover.removeRecursively(fileToRemove);
+ }
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java b/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
index a29bd4ac0cc1be9eebb311f708e5099bdc831376..452122c5650c0c0fea7960887ad0edaff8f1439d 100644
--- a/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
+++ b/common/source/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopier.java
@@ -429,7 +429,7 @@ public final class RsyncCopier implements IPathCopier, IDirectoryImmutableCopier
assert sourcePath != null;
assert sourceHostOrNull != null || sourcePath.exists() : logNonExistent(sourcePath);
assert destinationDirectory != null;
- assert destinationHostOrNull != null || destinationDirectory.isDirectory() : logNonExistent(sourcePath);
+ assert destinationHostOrNull != null || destinationDirectory.isDirectory() : logNonExistent(destinationDirectory);
// Only one side can be remote
assert sourceHostOrNull == null || destinationHostOrNull == null;
final List<String> commandLine =
diff --git a/common/source/java/ch/systemsx/cisd/common/process/CallableExecutor.java b/common/source/java/ch/systemsx/cisd/common/process/CallableExecutor.java
index 65d6cdda658bb3575540c1369c389ea8de7fa6d2..86a4aa18c0ee8914559ac9c2b0b80f98128db661 100644
--- a/common/source/java/ch/systemsx/cisd/common/process/CallableExecutor.java
+++ b/common/source/java/ch/systemsx/cisd/common/process/CallableExecutor.java
@@ -29,20 +29,20 @@ import ch.systemsx.cisd.common.exceptions.StopException;
*/
public final class CallableExecutor
{
- private final int maxInvocationsOnFailure;
+ private final int maxRetriesOnFailure;
private final long millisToSleepOnFailure;
public CallableExecutor()
{
- this(Constants.MAXIMUM_INVOCATIONS_ON_FAILURE, Constants.MILLIS_TO_SLEEP_BEFORE_RETRYING);
+ this(Constants.MAXIMUM_RETRY_COUNT, Constants.MILLIS_TO_SLEEP_BEFORE_RETRYING);
}
- public CallableExecutor(final int maxInvocationsOnFailure, final long millisToSleepOnFailure)
+ public CallableExecutor(final int maxRetriesOnFailure, final long millisToSleepOnFailure)
{
assert millisToSleepOnFailure > -1 : "Negative value";
- assert maxInvocationsOnFailure > -1 : "Negative value";
- this.maxInvocationsOnFailure = maxInvocationsOnFailure;
+ assert maxRetriesOnFailure > -1 : "Negative value";
+ this.maxRetriesOnFailure = maxRetriesOnFailure;
this.millisToSleepOnFailure = millisToSleepOnFailure;
}
@@ -63,12 +63,12 @@ public final class CallableExecutor
if (result == null)
{
++counter;
- if (counter < maxInvocationsOnFailure && millisToSleepOnFailure > 0)
+ if (counter < maxRetriesOnFailure && millisToSleepOnFailure > 0)
{
Thread.sleep(millisToSleepOnFailure);
}
}
- } while (result == null && counter < maxInvocationsOnFailure);
+ } while (result == null && counter < maxRetriesOnFailure);
} catch (final Exception ex)
{
throw CheckedExceptionTunnel.wrapIfNecessary(ex);
diff --git a/common/source/java/ch/systemsx/cisd/common/process/ProcessExecutionHelper.java b/common/source/java/ch/systemsx/cisd/common/process/ProcessExecutionHelper.java
index 23d8c03c34fe52f7071a0780709b54fa70abebad..9eeebac027bf4a055228c83086a6d5b8d8a1cff3 100644
--- a/common/source/java/ch/systemsx/cisd/common/process/ProcessExecutionHelper.java
+++ b/common/source/java/ch/systemsx/cisd/common/process/ProcessExecutionHelper.java
@@ -352,7 +352,10 @@ public final class ProcessExecutionHelper
}
} catch (final Exception ex)
{
- machineLog.error("Exception when launching: " + ex.getMessage());
+ machineLog.error(String
+ .format("Exception when launching: [%s, %s]",
+ ex.getClass().getSimpleName(), StringUtils.defaultIfEmpty(ex
+ .getMessage(), "NO MESSAGE")));
throw ex;
}
}
@@ -524,8 +527,7 @@ public final class ProcessExecutionHelper
private ExecutionResult<ProcessResult> getExecutionResult(
final Future<ProcessResult> runnerFuture)
{
- return ConcurrencyUtilities.getResult(runnerFuture, millisToWaitForCompletion, false, null,
- null);
+ return ConcurrencyUtilities.getResult(runnerFuture, millisToWaitForCompletion, false, null);
}
private Future<ProcessResult> launchProcessExecutor()
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/AbstractCopyActivitySensor.java b/common/source/java/ch/systemsx/cisd/common/utilities/AbstractCopyActivitySensor.java
index b1caa579be806ae443b6d3f99172ee8315fc97d2..4d4ff64712d8454e1c72043bec255d5d77e29d2e 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/AbstractCopyActivitySensor.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/AbstractCopyActivitySensor.java
@@ -19,17 +19,17 @@ package ch.systemsx.cisd.common.utilities;
import org.apache.commons.lang.time.DurationFormatUtils;
import org.apache.log4j.Logger;
-import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IActivitySensor;
+import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IDescribingActivitySensor;
import ch.systemsx.cisd.common.exceptions.StatusFlag;
import ch.systemsx.cisd.common.exceptions.StatusWithResult;
/**
- * A super class for {@link IActivitySensor}s that sense changes in some sort of copy operation to
- * a "target".
+ * A super class for {@link IDescribingActivitySensor}s that sense changes in some sort of copy
+ * operation to a "target".
*
* @author Bernd Rinn
*/
-public abstract class AbstractCopyActivitySensor implements IActivitySensor
+public abstract class AbstractCopyActivitySensor implements IDescribingActivitySensor
{
protected static final int DEFAULT_MAX_ERRORS_TO_IGNORE = 3;
@@ -72,7 +72,7 @@ public abstract class AbstractCopyActivitySensor implements IActivitySensor
* Returns a textual description of the target.
*/
protected abstract String getTargetDescription();
-
+
/**
* Returns the machine log for the concrete implementation.
*/
@@ -82,7 +82,7 @@ public abstract class AbstractCopyActivitySensor implements IActivitySensor
// IActivitySensor
//
- public long getTimeOfLastActivityMoreRecentThan(long thresholdMillis)
+ public long getLastActivityMillisMoreRecentThan(long thresholdMillis)
{
currentResult = getTargetTimeOfLastActivityMoreRecentThan(thresholdMillis);
final long now = System.currentTimeMillis();
@@ -92,13 +92,16 @@ public abstract class AbstractCopyActivitySensor implements IActivitySensor
if (errorCount <= maxErrorsToIgnore)
{
timeOfLastReportedActivity = now;
- getMachineLog().error(describeInactivity(now)
- + String.format(" (error count: %d <= %d, goes unreported)", errorCount,
- maxErrorsToIgnore));
+ getMachineLog().error(
+ describeInactivity(now)
+ + String.format(" (error count: %d <= %d, goes unreported)",
+ errorCount, maxErrorsToIgnore));
} else
{
- getMachineLog().error(describeInactivity(now)
- + String.format(" (error count: %s, reported to monitor)", errorCount));
+ getMachineLog().error(
+ describeInactivity(now)
+ + String.format(" (error count: %s, reported to monitor)",
+ errorCount));
}
} else
{
@@ -122,6 +125,16 @@ public abstract class AbstractCopyActivitySensor implements IActivitySensor
return timeOfLastReportedActivity;
}
+ public boolean hasActivityMoreRecentThan(long thresholdMillis)
+ {
+ final long now = System.currentTimeMillis();
+ return (now - getLastActivityMillisMoreRecentThan(thresholdMillis)) < thresholdMillis;
+ }
+
+ //
+ // IDescribingActivitySensor
+ //
+
public String describeInactivity(long now)
{
if (currentResult.isError())
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTask.java b/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTask.java
index e10f0ff37f6d7e87e108d2ecacaf162143bf1b35..8bc2c79c7e9d7018341fd83d3bf1fded86a0f134 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTask.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTask.java
@@ -32,6 +32,7 @@ import org.apache.log4j.Logger;
import ch.systemsx.cisd.common.collections.CollectionUtils;
import ch.systemsx.cisd.common.exceptions.Status;
+import ch.systemsx.cisd.common.exceptions.StopException;
import ch.systemsx.cisd.common.logging.ConditionalNotificationLogger;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
@@ -166,7 +167,7 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
final IPathHandler pathHandler, final int ignoredErrorCount)
{
this(asScannedStore(sourceDirectory, fileFilter), new FaultyPathDirectoryScanningHandler(
- sourceDirectory), PathHandlerAdapter
+ sourceDirectory, pathHandler), PathHandlerAdapter
.asScanningHandler(sourceDirectory, pathHandler), ignoredErrorCount);
}
@@ -234,7 +235,7 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
for (int i = 0; i < numberOfItems; i++)
{
final StoreItem storeItem = storeItems[i];
- if (stopRun)
+ if (stopRun || Thread.interrupted() || storeHandler.isStopped())
{
if (operationLog.isDebugEnabled())
{
@@ -262,6 +263,10 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
++numberOfItemsProcessedInLastRound;
} catch (final Exception ex)
{
+ if (ex instanceof StopException)
+ {
+ break;
+ }
// Do not stop when processing of one file has failed,
// continue with other files.
errorLog.put(storeItem, String.format(
@@ -300,7 +305,10 @@ public final class DirectoryScanningTimerTask extends TimerTask implements ITime
} while (numberOfItemsProcessedInLastRound > 0);
} catch (final Exception ex)
{
- printNotification(ex);
+ if (ex instanceof StopException == false)
+ {
+ printNotification(ex);
+ }
}
if (operationLog.isTraceEnabled())
{
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/FastHardLinkMaker.java b/common/source/java/ch/systemsx/cisd/common/utilities/FastHardLinkMaker.java
index 969a88462cf6275195b71a0ee1bb0c3e11578c53..87a4b12d04419149e38c9c57d8959b02476fc2a2 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/FastHardLinkMaker.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/FastHardLinkMaker.java
@@ -17,12 +17,11 @@
package ch.systemsx.cisd.common.utilities;
import java.io.File;
-import java.util.concurrent.Callable;
+import ch.systemsx.cisd.common.TimingParameters;
import ch.systemsx.cisd.common.concurrent.MonitoringProxy;
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
import ch.systemsx.cisd.common.filesystem.FileLinkUtilities;
-import ch.systemsx.cisd.common.filesystem.FileLinkUtilities.FileLinkException;
-import ch.systemsx.cisd.common.process.CallableExecutor;
/**
* A {@link IFileImmutableCopier} that uses a native method to create hard links.
@@ -45,7 +44,7 @@ public class FastHardLinkMaker implements IFileImmutableCopier
FileLinkUtilities.createHardLink(source.getAbsolutePath(), destination
.getAbsolutePath());
return true;
- } catch (FileLinkException ex)
+ } catch (WrappedIOException ex)
{
return false;
}
@@ -53,8 +52,8 @@ public class FastHardLinkMaker implements IFileImmutableCopier
};
/**
- * Returns <code>true</code>, if the native library could be initialized successfully and
- * thus this clase is operational, or <code>false</code> otherwise.
+ * Returns <code>true</code>, if the native library could be initialized successfully and thus
+ * this class is operational, or <code>false</code> otherwise.
*/
public final static boolean isOperational()
{
@@ -64,51 +63,51 @@ public class FastHardLinkMaker implements IFileImmutableCopier
/**
* Creates an {@link IFileImmutableCopier}.
*
- * @param millisToWaitForCompletion The time out in milli-seconds to wait for one link creation
- * to finish.
- * @param maxRetryOnFailure Maximal number of retries in case of failure.
- * @param millisToSleepOnFailure Time in milli-seconds to sleep after a failure has occured and
- * before trying it again.
+ * @param timingParameters The timing parameters used to monitor and potentially retry the hard
+ * link creation.
* @return The copier, if the native library could be initialized successfully, or
* <code>null</code> otherwise.
*/
- public final static IFileImmutableCopier tryCreate(final long millisToWaitForCompletion,
- final int maxRetryOnFailure, final long millisToSleepOnFailure)
+ public final static IFileImmutableCopier tryCreate(final TimingParameters timingParameters)
{
if (FileLinkUtilities.isOperational() == false)
{
return null;
}
- return new FastHardLinkMaker(millisToWaitForCompletion, maxRetryOnFailure,
- millisToSleepOnFailure);
+ return new FastHardLinkMaker(timingParameters);
}
- private final CallableExecutor retryingExecutor;
+ /**
+ * Creates an {@link IFileImmutableCopier} with default timing pameters (uses
+ * {@link TimingParameters#getDefaultParameters()}.
+ *
+ * @return The copier, if the native library could be initialized successfully, or
+ * <code>null</code> otherwise.
+ */
+ public final static IFileImmutableCopier tryCreate()
+ {
+ if (FileLinkUtilities.isOperational() == false)
+ {
+ return null;
+ }
+ return new FastHardLinkMaker(TimingParameters.getDefaultParameters());
+
+ }
private final IFileImmutableCopier monitoringProxy;
- private FastHardLinkMaker(final long millisToWaitForCompletion, final int maxRetryOnFailure,
- final long millisToSleepOnFailure)
+ private FastHardLinkMaker(final TimingParameters timingParameters)
{
- retryingExecutor = new CallableExecutor(maxRetryOnFailure, millisToSleepOnFailure);
monitoringProxy =
- MonitoringProxy.create(IFileImmutableCopier.class, nativeCopier).timeoutMillis(
- millisToWaitForCompletion).get();
+ MonitoringProxy.create(IFileImmutableCopier.class, nativeCopier).timing(
+ timingParameters).get();
}
public boolean copyFileImmutably(final File file, final File destinationDirectory,
final String nameOrNull)
{
- final Callable<Boolean> creatorCallable = new Callable<Boolean>()
- {
- public Boolean call() throws Exception
- {
- return monitoringProxy
- .copyFileImmutably(file, destinationDirectory, nameOrNull);
- }
- };
- return retryingExecutor.executeCallable(creatorCallable);
+ return monitoringProxy.copyFileImmutably(file, destinationDirectory, nameOrNull);
}
}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/FastRecursiveHardLinkMaker.java b/common/source/java/ch/systemsx/cisd/common/utilities/FastRecursiveHardLinkMaker.java
index 8c7a0f56703ab09b50b893851ffbd5cbe11886aa..af972a9093918bb71c94782a629352cc5a19bf07 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/FastRecursiveHardLinkMaker.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/FastRecursiveHardLinkMaker.java
@@ -20,6 +20,7 @@ import java.io.File;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.TimingParameters;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -38,80 +39,59 @@ public class FastRecursiveHardLinkMaker implements IImmutableCopier
private static final String RSYNC_EXEC = "rsync";
- private final static long MILLIS = 1000L;
-
- private final static long DEFAULT_INACTIVITY_TRESHOLD_MILLIS = 300 * MILLIS;
-
private final static int DEFAULT_MAX_ERRORS_TO_IGNORE = 3;
- private final static int DEFAULT_MAX_ATTEMPTS = 10;
-
- private final static long DEFAULT_TIME_TO_SLEEP_AFTER_COPY_FAILS = 5 * MILLIS;
-
private final IImmutableCopier fallbackCopierOrNull;
private final IFileImmutableCopier fastFileCopierOrNull;
private final IDirectoryImmutableCopier fastDirectoryCopierOrNull;
- public final static IImmutableCopier tryCreate(final File rsyncExecutable,
- boolean neverUseNative)
+ public final static IImmutableCopier tryCreate()
{
- return tryCreate(rsyncExecutable, DEFAULT_INACTIVITY_TRESHOLD_MILLIS, DEFAULT_MAX_ATTEMPTS,
- DEFAULT_TIME_TO_SLEEP_AFTER_COPY_FAILS, neverUseNative);
+ return tryCreate(TimingParameters.getDefaultParameters());
}
- public final static IImmutableCopier tryCreate(final File rsyncExecutable)
+ public final static IImmutableCopier tryCreate(final TimingParameters timingParameters)
{
- return tryCreate(rsyncExecutable, DEFAULT_INACTIVITY_TRESHOLD_MILLIS, DEFAULT_MAX_ATTEMPTS,
- DEFAULT_TIME_TO_SLEEP_AFTER_COPY_FAILS);
+ final File rsyncExecOrNull = OSUtilities.findExecutable(RSYNC_EXEC);
+ if (rsyncExecOrNull == null)
+ {
+ return null;
+ }
+ return create(rsyncExecOrNull, timingParameters);
}
- public final static IImmutableCopier tryCreate(final long millisToWaitForCompletion,
- final int maxRetryOnFailure, final long millisToSleepOnFailure)
+ public final static IImmutableCopier create(final File rsyncExecutable,
+ final TimingParameters parameters)
{
- return tryCreate(OSUtilities.findExecutable(RSYNC_EXEC), millisToWaitForCompletion,
- maxRetryOnFailure, millisToSleepOnFailure);
+ return create(rsyncExecutable, parameters, false);
}
- public final static IImmutableCopier tryCreate(final File rsyncExecutable,
- final long millisToWaitForCompletion, final int maxRetryOnFailure,
- final long millisToSleepOnFailure)
+ public final static IImmutableCopier create(final File rsyncExecutable)
{
- return tryCreate(rsyncExecutable, millisToWaitForCompletion, maxRetryOnFailure,
- millisToSleepOnFailure, false);
+ return create(rsyncExecutable, TimingParameters.getDefaultParameters(), false);
}
- public final static IImmutableCopier tryCreate(final File rsyncExecutable,
- final long millisToWaitForCompletion, final int maxRetryOnFailure,
- final long millisToSleepOnFailure, final boolean neverUseNative)
+ public final static IImmutableCopier create(final File rsyncExecutable,
+ final TimingParameters parameters, final boolean neverUseNative)
{
- try
- {
- return new FastRecursiveHardLinkMaker(rsyncExecutable, millisToSleepOnFailure,
- maxRetryOnFailure, millisToSleepOnFailure, neverUseNative);
- } catch (ConfigurationFailureException ex)
- {
- return null;
- }
+ return new FastRecursiveHardLinkMaker(rsyncExecutable, parameters, neverUseNative);
}
private FastRecursiveHardLinkMaker(final File rsyncExcutable,
- final long inactivityThresholdMillis, final int maxRetryOnFailure,
- final long millisToSleepOnFailure, final boolean neverUseNative)
+ final TimingParameters timingParameters, final boolean neverUseNative)
throws ConfigurationFailureException
{
this.fastFileCopierOrNull =
- neverUseNative ? null : FastHardLinkMaker.tryCreate(inactivityThresholdMillis,
- maxRetryOnFailure, millisToSleepOnFailure);
+ neverUseNative ? null : FastHardLinkMaker.tryCreate(timingParameters);
this.fastDirectoryCopierOrNull =
- new RsyncBasedRecursiveHardLinkMaker(rsyncExcutable, inactivityThresholdMillis,
- DEFAULT_MAX_ERRORS_TO_IGNORE, maxRetryOnFailure, millisToSleepOnFailure);
+ new RsyncBasedRecursiveHardLinkMaker(rsyncExcutable, timingParameters,
+ DEFAULT_MAX_ERRORS_TO_IGNORE);
if (fastFileCopierOrNull == null)
{
this.fallbackCopierOrNull =
- RecursiveHardLinkMaker.tryCreate(HardLinkMaker.tryCreateRetrying(
- inactivityThresholdMillis, maxRetryOnFailure, millisToSleepOnFailure));
+ RecursiveHardLinkMaker.tryCreate(HardLinkMaker.tryCreate(timingParameters));
} else
{
this.fallbackCopierOrNull = RecursiveHardLinkMaker.tryCreate(fastFileCopierOrNull);
@@ -123,6 +103,7 @@ public class FastRecursiveHardLinkMaker implements IImmutableCopier
}
if (operationLog.isInfoEnabled())
{
+ operationLog.info(timingParameters.toString());
if (fastFileCopierOrNull != null)
{
operationLog.info("Using native library to create hard link copies of files.");
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/FaultyPathDirectoryScanningHandler.java b/common/source/java/ch/systemsx/cisd/common/utilities/FaultyPathDirectoryScanningHandler.java
index b7e6c5d78b01d0a695c70beb7f416ff1b2cf1e22..a0f712b4c568eb2774bac6c9e6c4655661ea47b9 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/FaultyPathDirectoryScanningHandler.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/FaultyPathDirectoryScanningHandler.java
@@ -51,10 +51,14 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
private long faultyPathsLastChanged;
- public FaultyPathDirectoryScanningHandler(final File faultyPathDirectory)
+ private IStopSignaler stopSignaler;
+
+ public FaultyPathDirectoryScanningHandler(final File faultyPathDirectory,
+ final IStopSignaler stopSignaler)
{
this.faultyPaths = new HashSet<String>();
this.faultyPathsFile = new File(faultyPathDirectory, Constants.FAULTY_PATH_FILENAME);
+ this.stopSignaler = stopSignaler;
}
private final void checkForFaultyPathsFileChanged()
@@ -148,7 +152,7 @@ public final class FaultyPathDirectoryScanningHandler implements IDirectoryScann
{
// If the item still exists, we assume that it has not been handled. So it
// should be added to the faulty paths.
- if (scannedStore.existsOrError(storeItem))
+ if (scannedStore.existsOrError(storeItem) && stopSignaler.isStopped() == false)
{
addToFaultyPaths(scannedStore, storeItem);
return Status.createError("Failed to move item '%s'.", storeItem);
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/FileUtilities.java b/common/source/java/ch/systemsx/cisd/common/utilities/FileUtilities.java
index c6dd265ab01eb2ec2089197e2e512ca24ca6ab57..13344ac004d2eadd69a5c732739f52b63f7b7bdc 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/FileUtilities.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/FileUtilities.java
@@ -20,7 +20,6 @@ import java.io.BufferedReader;
import java.io.File;
import java.io.FileFilter;
import java.io.FileInputStream;
-import java.io.FileNotFoundException;
import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.FileWriter;
@@ -35,6 +34,7 @@ import java.text.DecimalFormat;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
+import java.util.LinkedList;
import java.util.List;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
@@ -45,11 +45,16 @@ import org.apache.commons.io.IOUtils;
import org.apache.commons.io.comparator.LastModifiedFileComparator;
import org.apache.commons.lang.CharUtils;
import org.apache.commons.lang.StringUtils;
+import org.apache.commons.lang.time.DurationFormatUtils;
+import ch.systemsx.cisd.common.concurrent.IActivityObserver;
+import ch.systemsx.cisd.common.concurrent.RecordingActivityObserverSensor;
+import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IDescribingActivitySensor;
import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.StopException;
import ch.systemsx.cisd.common.exceptions.UnknownLastChangedException;
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogLevel;
import ch.systemsx.cisd.common.parser.filter.AlwaysAcceptLineFilter;
@@ -64,7 +69,8 @@ import ch.systemsx.cisd.common.parser.filter.ILineFilter;
* <p>
* If you are tempted to add new functionality to this class, ensure that the new functionality does
* not yet exist in <code>org.apache.common.io.FileUtils</code>, see <a
- * href="http://jakarta.apache.org/commons/io/api-release/org/apache/commons/io/FileUtils.html">javadoc</a>.
+ * href="http://jakarta.apache.org/commons/io/api-release/org/apache/commons/io/FileUtils.html"
+ * >javadoc</a>.
*
* @author Bernd Rinn
*/
@@ -96,7 +102,7 @@ public final class FileUtilities
* @throws EnvironmentFailureException if a {@link IOException} occured.
*/
public static void copyFileTo(final File sourceFile, final File destinationFile,
- final boolean preservesLastModifiedDate) throws CheckedExceptionTunnel
+ final boolean preservesLastModifiedDate) throws WrappedIOException
{
FileInputStream inputStream = null;
FileOutputStream outputStream = null;
@@ -133,10 +139,10 @@ public final class FileUtilities
* is not <code>null</code>.
* @return The content of the file. All newline characters are '\n' (Unix convention). Never
* returns <code>null</code>.
- * @throws CheckedExceptionTunnel for wrapping an {@link IOException}, e.g. if the file does
- * not exist.
+ * @throws WrappedIOException for wrapping an {@link IOException}, e.g. if the file does not
+ * exist.
*/
- public static String loadToString(final File file) throws CheckedExceptionTunnel
+ public static String loadToString(final File file) throws WrappedIOException
{
assert file != null;
@@ -147,7 +153,7 @@ public final class FileUtilities
return readString(new BufferedReader(fileReader));
} catch (final IOException ex)
{
- throw new CheckedExceptionTunnel(ex);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
} finally
{
IOUtils.closeQuietly(fileReader);
@@ -161,10 +167,10 @@ public final class FileUtilities
* is not <code>null</code>.
* @return The content of the file. All newline characters are '\n' (Unix convention). Never
* returns <code>null</code>.
- * @throws CheckedExceptionTunnel for wrapping an {@link IOException}, e.g. if the file does
- * not exist.
+ * @throws WrappedIOException for wrapping an {@link IOException}, e.g. if the file does not
+ * exist.
*/
- public static String loadExactToString(final File file) throws CheckedExceptionTunnel
+ public static String loadExactToString(final File file) throws WrappedIOException
{
assert file != null;
@@ -175,7 +181,7 @@ public final class FileUtilities
return readExactString(new BufferedReader(fileReader));
} catch (final IOException ex)
{
- throw new CheckedExceptionTunnel(ex);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
} finally
{
IOUtils.closeQuietly(fileReader);
@@ -185,9 +191,9 @@ public final class FileUtilities
/**
* Writes the specified string to the specified file.
*
- * @throws CheckedExceptionTunnel for wrapping an {@link IOException}.
+ * @throws WrappedIOException for wrapping an {@link IOException}.
*/
- public static void writeToFile(final File file, final String str) throws CheckedExceptionTunnel
+ public static void writeToFile(final File file, final String str) throws WrappedIOException
{
assert file != null : "Unspecified file.";
assert str != null : "Unspecified string.";
@@ -199,7 +205,7 @@ public final class FileUtilities
fileWriter.write(str);
} catch (final IOException ex)
{
- throw new CheckedExceptionTunnel(ex);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
} finally
{
IOUtils.closeQuietly(fileWriter);
@@ -212,11 +218,10 @@ public final class FileUtilities
* @param file the file that should be loaded. This method asserts that given <code>File</code>
* is not <code>null</code>.
* @return The content of the file line by line.
- * @throws CheckedExceptionTunnel for wrapping an {@link IOException}, e.g. if the file does
- * not exist.
+ * @throws WrappedIOException for wrapping an {@link IOException}, e.g. if the file does not
+ * exist.
*/
- public final static List<String> loadToStringList(final File file)
- throws CheckedExceptionTunnel
+ public final static List<String> loadToStringList(final File file) throws WrappedIOException
{
return loadToStringList(file, null);
}
@@ -229,11 +234,11 @@ public final class FileUtilities
* @param lineFilterOrNull a line filter if you are not interested in all lines. May be
* <code>null</code>.
* @return The content of the file line by line.
- * @throws CheckedExceptionTunnel for wrapping an {@link IOException}, e.g. if the file does
- * not exist.
+ * @throws WrappedIOException for wrapping an {@link IOException}, e.g. if the file does not
+ * exist.
*/
public final static List<String> loadToStringList(final File file,
- final ILineFilter lineFilterOrNull) throws CheckedExceptionTunnel
+ final ILineFilter lineFilterOrNull) throws WrappedIOException
{
assert file != null : "Unspecified file.";
@@ -244,7 +249,7 @@ public final class FileUtilities
return readStringList(new BufferedReader(fileReader), lineFilterOrNull);
} catch (final IOException ex)
{
- throw new CheckedExceptionTunnel(ex);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
} finally
{
IOUtils.closeQuietly(fileReader);
@@ -261,12 +266,12 @@ public final class FileUtilities
* <code>null</code>).
* @param resource Absolute path of the resource (will be the argument of
* <code>getResource()</code>).
- * @return The content of the resource, or <code>null</code> if the specified resource does
- * not exist.
- * @throws CheckedExceptionTunnel for wrapping an {@link IOException}
+ * @return The content of the resource, or <code>null</code> if the specified resource does not
+ * exist.
+ * @throws WrappedIOException for wrapping an {@link IOException}
*/
public static String loadToString(final Class<?> clazz, final String resource)
- throws CheckedExceptionTunnel
+ throws WrappedIOException
{
assert clazz != null : "Given class can not be null.";
assert resource != null && resource.length() > 0 : "Given resource can not be null.";
@@ -278,7 +283,7 @@ public final class FileUtilities
return reader == null ? null : readString(reader);
} catch (final IOException ex)
{
- throw new CheckedExceptionTunnel(ex);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
} finally
{
IOUtils.closeQuietly(reader);
@@ -295,11 +300,11 @@ public final class FileUtilities
* @param resource absolute path of the resource (will be the argument of
* <code>getResource()</code>).
* @return The content of the resource line by line.
- * @throws CheckedExceptionTunnel for wrapping an {@link IOException}, e.g. if the file does
- * not exist.
+ * @throws WrappedIOException for wrapping an {@link IOException}, e.g. if the file does not
+ * exist.
*/
public final static List<String> loadToStringList(final Class<?> clazz, final String resource)
- throws CheckedExceptionTunnel
+ throws WrappedIOException
{
return loadToStringList(clazz, resource, null);
}
@@ -317,11 +322,11 @@ public final class FileUtilities
* <code>null</code>.
* @return The content of the resource line by line or <code>null</code> if given
* <var>resource</var> can not be found.
- * @throws CheckedExceptionTunnel for wrapping an {@link IOException}, e.g. if the file does
- * not exist.
+ * @throws WrappedIOException for wrapping an {@link IOException}, e.g. if the file does not
+ * exist.
*/
public final static List<String> loadToStringList(final Class<?> clazz, final String resource,
- final ILineFilter lineFilterOrNull) throws CheckedExceptionTunnel
+ final ILineFilter lineFilterOrNull) throws WrappedIOException
{
assert clazz != null : "Given class can not be null.";
assert StringUtils.isNotEmpty(resource) : "Given resource can not be empty.";
@@ -333,7 +338,7 @@ public final class FileUtilities
return reader == null ? null : readStringList(reader, lineFilterOrNull);
} catch (final IOException ex)
{
- throw new CheckedExceptionTunnel(ex);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
} finally
{
IOUtils.closeQuietly(reader);
@@ -341,7 +346,7 @@ public final class FileUtilities
}
private final static BufferedReader tryGetBufferedReader(final Class<?> clazz,
- final String resource) throws FileNotFoundException
+ final String resource)
{
final URL url = clazz.getResource(resource);
if (url == null)
@@ -351,9 +356,12 @@ public final class FileUtilities
try
{
return new BufferedReader(new FileReader(new File(url.toURI())));
+ } catch (final IOException ex)
+ {
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
} catch (final URISyntaxException ex)
{
- throw new CheckedExceptionTunnel(ex);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
}
}
@@ -407,8 +415,7 @@ public final class FileUtilities
}
/**
- * Checks whether a <var>path</var> of some <var>kind</var> is fully accessible to the
- * program.
+ * Checks whether a <var>path</var> of some <var>kind</var> is fully accessible to the program.
*
* @param kindOfPath description of given <var>path</var>. Mainly used for error messages.
* @return <code>null</code> if the <var>directory</var> is fully accessible and an error
@@ -442,8 +449,8 @@ public final class FileUtilities
* Checks whether a <var>directory</var> of some <var>kind</var> is accessible for reading to
* the program (it's a directory, you can read and write in it)
*
- * @return <code>null</code> if the <var>directory</var> is accessible for reading and an
- * error message describing the problem with the <var>directory</var> otherwise.
+ * @return <code>null</code> if the <var>directory</var> is accessible for reading and an error
+ * message describing the problem with the <var>directory</var> otherwise.
*/
public static String checkDirectoryReadAccessible(final File directory,
final String kindOfDirectory)
@@ -504,8 +511,8 @@ public final class FileUtilities
}
/**
- * Checks whether a <var>file</var> of some <var>kindOfFile</var> is accessible for reading
- * and writing to the program (so it's a file and you can read and write it)
+ * Checks whether a <var>file</var> of some <var>kindOfFile</var> is accessible for reading and
+ * writing to the program (so it's a file and you can read and write it)
*
* @return <code>null</code> if the <var>file</var> is fully accessible and an error message
* describing the problem with the <var>file</var> otherwise.
@@ -552,14 +559,25 @@ public final class FileUtilities
}
/**
- * A simple role that can observe activity.
+ * A class that monitors activity on recursive delete processes.
*/
- public interface SimpleActivityObserver
+ public static class DeleteActivityDetector extends RecordingActivityObserverSensor implements
+ IDescribingActivitySensor
{
- /**
- * Called when activity occurred.
- */
- public void update();
+ private final File path;
+
+ public DeleteActivityDetector(File path)
+ {
+ super();
+ this.path = path;
+ }
+
+ public String describeInactivity(long now)
+ {
+ return "No delete activity of path " + path.getPath() + " for "
+ + DurationFormatUtils.formatDurationHMS(now - getLastActivityMillis());
+ }
+
}
/**
@@ -570,10 +588,11 @@ public final class FileUtilities
* <code>null</code>.
*
* @param path Path of the file or directory to delete.
- * @return <code>true</code> if the path has been delete successfully, <code>false</code>
+ * @return <code>true</code> if the path has been deleted successfully, <code>false</code>
* otherwise.
+ * @throws StopException If the current thread has been interrupted.
*/
- public static boolean deleteRecursively(final File path)
+ public static boolean deleteRecursively(final File path) throws StopException
{
assert path != null;
@@ -587,10 +606,12 @@ public final class FileUtilities
* @param path Path of the file or directory to delete.
* @param loggerOrNull The logger that should be used to log deletion of path entries, or
* <code>null</code> if nothing should be logged.
- * @return <code>true</code> if the path has been delete successfully, <code>false</code>
+ * @return <code>true</code> if the path has been deleted successfully, <code>false</code>
* otherwise.
+ * @throws StopException If the current thread has been interrupted.
*/
public static boolean deleteRecursively(final File path, final ISimpleLogger loggerOrNull)
+ throws StopException
{
return deleteRecursively(path, loggerOrNull, null);
}
@@ -602,15 +623,16 @@ public final class FileUtilities
* @param path Path of the file or directory to delete.
* @param loggerOrNull The logger that should be used to log deletion of path entries, or
* <code>null</code> if nothing should be logged.
- * @param observerOrNull If not <code>null</code>, will be updated on progress in the
- * deletion. This can be used to find out whether a (potentially) long-running
- * recursive deletion call is alive-and-kicking or hangs (e.g. due to a remote
- * directory becoming unresponsive).
- * @return <code>true</code> if the path has been delete successfully, <code>false</code>
+ * @param observerOrNull If not <code>null</code>, will be updated on progress in the deletion.
+ * This can be used to find out whether a (potentially) long-running recursive
+ * deletion call is alive-and-kicking or hangs (e.g. due to a remote directory
+ * becoming unresponsive).
+ * @return <code>true</code> if the path has been deleted successfully, <code>false</code>
* otherwise.
+ * @throws StopException If the current thread has been interrupted.
*/
public static boolean deleteRecursively(final File path, final ISimpleLogger loggerOrNull,
- final SimpleActivityObserver observerOrNull)
+ final IActivityObserver observerOrNull) throws StopException
{
assert path != null;
@@ -618,6 +640,10 @@ public final class FileUtilities
{
for (final File file : path.listFiles())
{
+ if (Thread.interrupted())
+ {
+ throw new StopException();
+ }
if (observerOrNull != null)
{
observerOrNull.update();
@@ -655,9 +681,10 @@ public final class FileUtilities
* @param logger The logger that should be used to log deletion of path entries, or
* <code>null</code> if nothing should be logged.
* @return <code>true</code> if the <var>path</var> itself has been deleted.
+ * @throws StopException If the current thread has been interrupted.
*/
public static boolean deleteRecursively(final File path, final FileFilter filter,
- final ISimpleLogger logger)
+ final ISimpleLogger logger) throws StopException
{
assert path != null;
@@ -674,14 +701,16 @@ public final class FileUtilities
* @param filter The {@link FileFilter} to use when deciding which paths to delete.
* @param logger The logger that should be used to log deletion of path entries, or
* <code>null</code> if nothing should be logged.
- * @param observerOrNull If not <code>null</code>, will be updated on progress in the
- * deletion. This can be used to find out whether a (potentially) long-running
- * recursive deletion call is alive-and-kicking or hangs (e.g. due to a remote
- * directory becoming unresponsive).
+ * @param observerOrNull If not <code>null</code>, will be updated on progress in the deletion.
+ * This can be used to find out whether a (potentially) long-running recursive
+ * deletion call is alive-and-kicking or hangs (e.g. due to a remote directory
+ * becoming unresponsive).
* @return <code>true</code> if the <var>path</var> itself has been deleted.
+ * @throws StopException If the current thread has been interrupted.
*/
public static boolean deleteRecursively(final File path, final FileFilter filter,
- final ISimpleLogger logger, final SimpleActivityObserver observerOrNull)
+ final ISimpleLogger logger, final IActivityObserver observerOrNull)
+ throws StopException
{
assert path != null;
assert filter != null;
@@ -695,6 +724,10 @@ public final class FileUtilities
{
for (final File file : path.listFiles())
{
+ if (Thread.interrupted())
+ {
+ throw new StopException();
+ }
if (observerOrNull != null)
{
observerOrNull.update();
@@ -813,22 +846,21 @@ public final class FileUtilities
}
/**
- * Determines the time (in milliseconds since start of the epoch) when any item below <var>path</var>
- * has last been changed in the file system.
+ * Determines the time (in milliseconds since start of the epoch) when any item below
+ * <var>path</var> has last been changed in the file system.
*
* @param path The path (file or directory) to check for last change.
- * @param subDirectoriesOnly If <code>true</code>, only subdirectories of <var>path</var>
- * are checked, if <var>path</var> is a directory. If <var>path</var> is a file,
- * this parameter is ignored. When considering what this parameter is good for, note
- * that the mtime of a directory is changed when an entry in the directory changes.
+ * @param subDirectoriesOnly If <code>true</code>, only subdirectories of <var>path</var> are
+ * checked, if <var>path</var> is a directory. If <var>path</var> is a file, this
+ * parameter is ignored. When considering what this parameter is good for, note that
+ * the mtime of a directory is changed when an entry in the directory changes.
* @param stopWhenFindYounger If > 0, the recursive search for younger file will be stopped
* when a file or directory is found that is younger than the time specified in this
* parameter. Supposed to be used when one does not care about the absolute youngest
* entry, but only, if there are entries that are "young enough".
* @return The time when any file in (or below) <var>path</var> has last been changed in the
* file system.
- * @throws UnknownLastChangedException if the <var>path</var> does not exist or is not
- * readable.
+ * @throws UnknownLastChangedException if the <var>path</var> does not exist or is not readable.
* @throws StopException if the thread that the method runs in gets interrupted.
*/
public static long lastChanged(final File path, final boolean subDirectoriesOnly,
@@ -839,23 +871,22 @@ public final class FileUtilities
}
/**
- * Determines the time (in milliseconds since start of the epoch) when any item below <var>path</var>
- * has last been changed in the file system.
+ * Determines the time (in milliseconds since start of the epoch) when any item below
+ * <var>path</var> has last been changed in the file system.
*
* @param path The path (file or directory) to check for last change.
- * @param subDirectoriesOnly If <code>true</code>, only subdirectories of <var>path</var>
- * are checked, if <var>path</var> is a directory. If <var>path</var> is a file,
- * this parameter is ignored. When considering what this parameter is good for, note
- * that the mtime of a directory is changed when an entry in the directory changes.
+ * @param subDirectoriesOnly If <code>true</code>, only subdirectories of <var>path</var> are
+ * checked, if <var>path</var> is a directory. If <var>path</var> is a file, this
+ * parameter is ignored. When considering what this parameter is good for, note that
+ * the mtime of a directory is changed when an entry in the directory changes.
* @param stopWhenFindYoungerRelative If > 0, the recursive search for younger file will be
* stopped when a file or directory is found that is younger than
- * <code>System.currentTimeMillis() - stopWhenYoungerRelative</code>. Supposed to
- * be used when one does not care about the absolute youngest entry, but only, if
- * there are entries that are "young enough".
+ * <code>System.currentTimeMillis() - stopWhenYoungerRelative</code>. Supposed to be
+ * used when one does not care about the absolute youngest entry, but only, if there
+ * are entries that are "young enough".
* @return The time when any file in (or below) <var>path</var> has last been changed in the
* file system.
- * @throws UnknownLastChangedException if the <var>path</var> does not exist or is not
- * readable.
+ * @throws UnknownLastChangedException if the <var>path</var> does not exist or is not readable.
* @throws StopException if the thread that the method runs in gets interrupted.
*/
public static long lastChangedRelative(final File path, final boolean subDirectoriesOnly,
@@ -868,8 +899,7 @@ public final class FileUtilities
/**
* @return The time when any file in (or below) <var>path</var> has last been changed in the
* file system.
- * @throws UnknownLastChangedException if the <var>path</var> does not exist or is not
- * readable.
+ * @throws UnknownLastChangedException if the <var>path</var> does not exist or is not readable.
* @throws StopException if the thread that the method runs in gets interrupted.
*/
public static long lastChanged(final File path) throws UnknownLastChangedException
@@ -919,9 +949,9 @@ public final class FileUtilities
*
* @param defaultFileNameOrNull the default name for the new file if the digit pattern could not
* be found in its name. If empty then "1" will be appended to its name.
- * @param regexOrNull pattern to find out the counter. If <code>null</code> then a default (<code>(\\d+)</code>)
- * will be used. The given <var>regex</var> must contain <code>(\\d+)</code> or
- * <code>([0-9]+)</code>.
+ * @param regexOrNull pattern to find out the counter. If <code>null</code> then a default (
+ * <code>(\\d+)</code>) will be used. The given <var>regex</var> must contain
+ * <code>(\\d+)</code> or <code>([0-9]+)</code>.
*/
public final static File createNextNumberedFile(final File path, final Pattern regexOrNull,
final String defaultFileNameOrNull)
@@ -1006,8 +1036,8 @@ public final class FileUtilities
*
* @param directory the directory to list
* @param loggerOrNull logger, if <code>null</code> than no logging occurs
- * @return all files in <var>directory</var> or <code>null</code>, if <var>directory</var>
- * does not exist or is not a directory.
+ * @return all files in <var>directory</var> or <code>null</code>, if <var>directory</var> does
+ * not exist or is not a directory.
*/
public static File[] tryListFiles(final File directory, final ISimpleLogger loggerOrNull)
{
@@ -1019,7 +1049,7 @@ public final class FileUtilities
*
* @param directory the directory to list
* @param filterOrNull only files matching this filter will show up in the result, if it is not
- * <code>null</code>.
+ * <code>null</code>
* @param loggerOrNull logger, if <code>null</code> than no logging occurs
* @return all files in <var>directory</var> that match the filter, or <code>null</code>, if
* <var>directory</var> does not exist or is not a directory.
@@ -1081,10 +1111,10 @@ public final class FileUtilities
* @param postfix The postfix to use for the temporary name.
* @return The name of the temporary file.
* @throws IllegalArgumentException If the resource cannot be found in the class path.
- * @throws CheckedExceptionTunnel If an {@link IOException} occurs.
+ * @throws WrappedIOException If an {@link IOException} occurs.
*/
public final static String copyResourceToTempFile(final String resource, final String prefix,
- final String postfix)
+ final String postfix) throws WrappedIOException
{
final InputStream resourceStream = FileUtilities.class.getResourceAsStream(resource);
if (resourceStream == null)
@@ -1106,7 +1136,7 @@ public final class FileUtilities
return tempFile.getAbsolutePath();
} catch (final IOException ex)
{
- throw new CheckedExceptionTunnel(ex);
+ throw CheckedExceptionTunnel.wrapIfNecessary(ex);
} finally
{
IOUtils.closeQuietly(resourceStream);
@@ -1133,8 +1163,8 @@ public final class FileUtilities
* Loads the native library <var>libraryName</var> from a Java resource that follows tha naming
* convention described in {@link #tryCopyNativeLibraryToTempFile(String)}.
*
- * @return <code>true</code> if the library has been loaded successfully and
- * <code>false</code> otherwise.
+ * @return <code>true</code> if the library has been loaded successfully and <code>false</code>
+ * otherwise.
*/
public final static boolean loadNativeLibraryFromResource(final String libraryName)
{
@@ -1200,6 +1230,226 @@ public final class FileUtilities
return fileList;
}
+ /**
+ * A {@link FileFilter} that matches against a list of file extensions and that
+ *
+ * @author Bernd Rinn
+ */
+ private static final class ExtensionFileFilter implements FileFilter
+ {
+ private final String[] extensionsOrNull;
+
+ private final IActivityObserver observerOrNull;
+
+ private final boolean recursive;
+
+ ExtensionFileFilter(String[] extensionsOrNull, boolean recursive,
+ IActivityObserver observerOrNull)
+ {
+ this.extensionsOrNull = extensionsOrNull;
+ this.recursive = recursive;
+ this.observerOrNull = observerOrNull;
+ }
+
+ private boolean correctType(File file)
+ {
+ // Small optimization: if recursive, we know alrady that file.isDirectory() == false.
+ return recursive || file.isFile();
+ }
+
+ private boolean match(String extensionFound)
+ {
+ if (extensionsOrNull == null)
+ {
+ return true;
+ }
+ if (extensionFound.length() == 0)
+ {
+ return false;
+ }
+ for (String ext : extensionsOrNull)
+ {
+ if (extensionFound.equals(ext))
+ {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean accept(File file)
+ {
+ if (observerOrNull != null)
+ {
+ observerOrNull.update();
+ }
+ if (recursive && file.isDirectory())
+ {
+ return true; // We need to traverse directories in any case.
+ }
+ return correctType(file) && match(FilenameUtils.getExtension(file.getName()));
+ }
+
+ }
+
+ /**
+ * A {@link FileFilter} that matches against a list of file extensions and that
+ *
+ * @author Bernd Rinn
+ */
+ private static final class DirectoryFilter implements FileFilter
+ {
+ private final IActivityObserver observerOrNull;
+
+ DirectoryFilter(IActivityObserver observerOrNull)
+ {
+ this.observerOrNull = observerOrNull;
+ }
+
+ public boolean accept(File pathname)
+ {
+ if (observerOrNull != null)
+ {
+ observerOrNull.update();
+ }
+ return pathname.isDirectory();
+ }
+
+ }
+
+ /**
+ * A {@link FileFilter} that matches against a list of file extensions and that
+ *
+ * @author Bernd Rinn
+ */
+ private static final class TrueFilter implements FileFilter
+ {
+ private final IActivityObserver observerOrNull;
+
+ TrueFilter(IActivityObserver observerOrNull)
+ {
+ this.observerOrNull = observerOrNull;
+ }
+
+ public boolean accept(File pathname)
+ {
+ if (observerOrNull != null)
+ {
+ observerOrNull.update();
+ }
+ return true;
+ }
+
+ }
+
+ /**
+ * Finds files within a given directory (and optionally its subdirectories) which match an array
+ * of extensions.
+ *
+ * @param directory The directory to search in.
+ * @param extensionsOrNull An array of extensions, ex. {"java","xml"}. If this parameter is
+ * <code>null</code>, all files are returned.
+ * @param recursive If true all subdirectories are searched as well.
+ * @param observerOrNull If not <code>null</code>, will be updated on progress of file
+ * gathering. This can be used to find out whether a (potentially) long-running file
+ * gathering call is alive-and-kicking or hangs (e.g. due to a remote directory
+ * becoming unresponsive).
+ * @return A list of java.io.File (all files) with the matching files, or an empty list, if
+ * <var>directory</var> ist not a directory.
+ */
+ public static List<File> listFiles(File directory, String[] extensionsOrNull,
+ boolean recursive, IActivityObserver observerOrNull)
+ {
+ assert directory != null;
+
+ final List<File> result = new LinkedList<File>();
+ internalListFiles(directory, result, new ExtensionFileFilter(extensionsOrNull, recursive,
+ observerOrNull), observerOrNull, recursive, FType.FILE);
+ return result;
+ }
+
+ /**
+ * Finds directories within a given directory (and optionally its subdirectories).
+ *
+ * @param directory The directory to search in.
+ * @param recursive If true all subdirectories are searched as well.
+ * @param observerOrNull If not <code>null</code>, will be updated on progress of file
+ * gathering. This can be used to find out whether a (potentially) long-running file
+ * gathering call is alive-and-kicking or hangs (e.g. due to a remote directory
+ * becoming unresponsive).
+ * @return A list of java.io.File (all directories), or an empty list, if <var>directory</var>
+ * ist not a directory.
+ */
+ public static List<File> listDirectories(File directory, boolean recursive,
+ IActivityObserver observerOrNull)
+ {
+ assert directory != null;
+
+ final List<File> result = new LinkedList<File>();
+ internalListFiles(directory, result, new DirectoryFilter(observerOrNull), observerOrNull,
+ recursive, FType.DIRECTORY);
+ return result;
+ }
+
+ /**
+ * Finds files and directories within a given directory (and optionally its subdirectories).
+ *
+ * @param directory The directory to search in.
+ * @param recursive If true all subdirectories are searched as well.
+ * @param observerOrNull If not <code>null</code>, will be updated on progress of file
+ * gathering. This can be used to find out whether a (potentially) long-running file
+ * gathering call is alive-and-kicking or hangs (e.g. due to a remote directory
+ * becoming unresponsive).
+ * @return A list of java.io.File (all directories), or an empty list, if <var>directory</var>
+ * ist not a directory.
+ */
+ public static List<File> listFilesAndDirectories(File directory, boolean recursive,
+ IActivityObserver observerOrNull)
+ {
+ assert directory != null;
+
+ final List<File> result = new LinkedList<File>();
+ internalListFiles(directory, result, new TrueFilter(observerOrNull), observerOrNull,
+ recursive, FType.EITHER);
+ return result;
+ }
+
+ private enum FType
+ {
+ FILE, DIRECTORY, EITHER
+ }
+
+ private static void internalListFiles(File directory, List<File> result, FileFilter filter,
+ IActivityObserver observerOrNull, boolean recursive, FType ftype)
+ {
+ final File[] filteredFilesAndDirectories = directory.listFiles(filter);
+ if (filteredFilesAndDirectories == null)
+ {
+ return;
+ }
+ for (File f : filteredFilesAndDirectories)
+ {
+ if (observerOrNull != null)
+ {
+ observerOrNull.update();
+ }
+ if (f.isDirectory())
+ {
+ if (ftype != FType.FILE)
+ {
+ result.add(f);
+ }
+ if (recursive)
+ {
+ internalListFiles(f, result, filter, observerOrNull, recursive, ftype);
+ }
+ } else if (ftype != FType.DIRECTORY)
+ {
+ result.add(f);
+ }
+ }
+ }
+
/**
* Normalizes given <var>file</var> path, removing double and single dot path steps.
* <p>
@@ -1243,8 +1493,8 @@ public final class FileUtilities
* Returns a human-readable version of the file size, where the input represents a specific
* number of bytes.
* <p>
- * By comparison with {@link FileUtils#byteCountToDisplaySize(long)}, the output of this
- * version is more exact.
+ * By comparison with {@link FileUtils#byteCountToDisplaySize(long)}, the output of this version
+ * is more exact.
* </p>
*
* @param size the number of bytes
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/HardLinkMaker.java b/common/source/java/ch/systemsx/cisd/common/utilities/HardLinkMaker.java
index 755384d207655fb6a1a8cb18d86bb24040f8f77f..104dea9df03d5f1d7ed2be496e3f2c825f49739b 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/HardLinkMaker.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/HardLinkMaker.java
@@ -25,6 +25,7 @@ import java.util.concurrent.Callable;
import org.apache.commons.io.FileUtils;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.TimingParameters;
import ch.systemsx.cisd.common.exceptions.StopException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -32,6 +33,8 @@ import ch.systemsx.cisd.common.process.CallableExecutor;
import ch.systemsx.cisd.common.process.ProcessExecutionHelper;
/**
+ * A class for creating hard links based on the Unix 'ln' program.
+ *
* @author Bernd Rinn
*/
public class HardLinkMaker implements IFileImmutableCopier
@@ -46,111 +49,80 @@ public class HardLinkMaker implements IFileImmutableCopier
private final String linkExecPath;
- private final RetryingOperationTimeout singleFileLinkTimeout;
-
- private static class RetryingOperationTimeout
- {
- private final long millisToWaitForCompletion;
-
- private final int maxRetryOnFailure;
-
- private final long millisToSleepOnFailure;
-
- private RetryingOperationTimeout(long millisToWaitForCompletion, int maxRetryOnFailure,
- long millisToSleepOnFailure)
- {
- this.millisToWaitForCompletion = millisToWaitForCompletion;
- this.maxRetryOnFailure = maxRetryOnFailure;
- this.millisToSleepOnFailure = millisToSleepOnFailure;
- }
-
- public long getMillisToWaitForCompletion()
- {
- return millisToWaitForCompletion;
- }
-
- public int getMaxRetryOnFailure()
- {
- return maxRetryOnFailure;
- }
-
- public long getMillisToSleepOnFailure()
- {
- return millisToSleepOnFailure;
- }
- }
-
- private HardLinkMaker(final String linkExecPath,
- RetryingOperationTimeout singleFileLinkTimeoutOrNull)
- {
- this.linkExecPath = linkExecPath;
- this.singleFileLinkTimeout =
- singleFileLinkTimeoutOrNull != null ? singleFileLinkTimeoutOrNull
- : createNoTimeout();
- }
+ private final TimingParameters timingParameters;
//
// Factory methods
//
/**
- * Creates copier which won't retry an operation if it fails.
+ * Creates copier trying to find the path to the <code>ln</code> executable.
*
- * @param linkExecPath The path to the <code>ln</code> executable.
+ * @param timingParameters The timing parameters to use when monitoring the call to 'ln'.
+ * @return <code>null</code> if the <code>ln</code> executable was not found.
*/
- public static final IFileImmutableCopier create(final String linkExecPath)
+ public static final IFileImmutableCopier tryCreate(TimingParameters timingParameters)
{
- return new HardLinkMaker(linkExecPath, null);
+ final File lnExec = OSUtilities.findExecutable(HARD_LINK_EXEC);
+ if (lnExec == null)
+ {
+ return null;
+ }
+ return create(lnExec, timingParameters);
}
/**
- * Creates copier trying to find the path to the <code>ln</code> executable.
+ * Creates copier trying to find the path to the <code>ln</code> executable. Uses default timing
+ * parameters.
*
* @return <code>null</code> if the <code>ln</code> executable was not found.
+ * @see TimingParameters#getDefaultParameters()
*/
public static final IFileImmutableCopier tryCreate()
{
- return tryCreate(null);
+ final File lnExec = OSUtilities.findExecutable(HARD_LINK_EXEC);
+ if (lnExec == null)
+ {
+ return null;
+ }
+ return create(lnExec);
}
/**
- * Creates copier which is able to retry the operation of creating the hard link of a file if
- * it does not complete after a specified timeout.
+ * Creates copier which is able to retry the operation of creating the hard link of a file if it
+ * does not complete after a specified timeout.
*
- * @param millisToWaitForCompletion The time to wait for the process creating one hard link to a
- * file to complete in milli seconds. If the process is not finished after that time,
- * it will be terminated.
- * @param maxRetryOnFailure The number of times we should try to create each hard link if copy
- * operation fails.
- * @param millisToSleepOnFailure The number of milliseconds we should wait before re-executing
- * the copy of a single file. Specify 0 to wait till the first operation completes.
+ * @param lnExec The executable of the 'ln' program.
+ * @param timingParameters The timing parameters to use when monitoring the call to 'ln'.
*/
- public static final IFileImmutableCopier tryCreateRetrying(
- final long millisToWaitForCompletion, final int maxRetryOnFailure,
- final long millisToSleepOnFailure)
+ public static final IFileImmutableCopier create(final File lnExec,
+ final TimingParameters timingParameters)
{
- RetryingOperationTimeout timeout =
- new RetryingOperationTimeout(millisToWaitForCompletion, maxRetryOnFailure,
- millisToSleepOnFailure);
- return tryCreate(timeout);
+ return new HardLinkMaker(lnExec.getAbsolutePath(), timingParameters);
}
- private static final IFileImmutableCopier tryCreate(
- RetryingOperationTimeout singleFileLinkTimeoutOrNull)
+ /**
+ * Creates copier which is able to retry the operation of creating the hard link of a file if it
+ * does not complete after a specified timeout. Uses default timing parameters.
+ *
+ * @param lnExec The executable of the 'ln' program.
+ * @see TimingParameters#getDefaultParameters()
+ */
+ public static final IFileImmutableCopier create(final File lnExec)
{
- final File lnExec = OSUtilities.findExecutable(HARD_LINK_EXEC);
- if (lnExec == null)
- {
- return null;
- }
- return new HardLinkMaker(lnExec.getAbsolutePath(), singleFileLinkTimeoutOrNull);
+ return new HardLinkMaker(lnExec.getAbsolutePath(), TimingParameters.getDefaultParameters());
}
- private RetryingOperationTimeout createNoTimeout()
+ private HardLinkMaker(final String linkExecPath, TimingParameters timingParameters)
{
- return new RetryingOperationTimeout(0, 1, 0);
+ this.linkExecPath = linkExecPath;
+ this.timingParameters = timingParameters;
}
+ //
+ // IFileImutableCopier
+ //
+
public boolean copyFileImmutably(final File source, final File destinationDirectory,
final String nameOrNull)
{
@@ -165,9 +137,9 @@ public class HardLinkMaker implements IFileImmutableCopier
{
boolean result =
ProcessExecutionHelper.runAndLog(cmd, operationLog, machineLog,
- singleFileLinkTimeout.getMillisToWaitForCompletion());
- // NOTE: we have noticed that sometimes the result is false although the file
- // have been copied
+ timingParameters.getTimeoutMillis());
+ // NOTE: we have noticed that in some environments sometimes the result is
+ // false although the file have been copied
if (result == false && destFile.exists()
&& checkIfIdenticalContent(source, destFile))
{
@@ -182,8 +154,8 @@ public class HardLinkMaker implements IFileImmutableCopier
}
};
boolean ok =
- runRepeatableProcess(processTask, singleFileLinkTimeout.getMaxRetryOnFailure(),
- singleFileLinkTimeout.getMillisToSleepOnFailure());
+ runRepeatableProcess(processTask, timingParameters.getMaxRetriesOnFailure(),
+ timingParameters.getIntervalToWaitAfterFailureMillis());
return ok;
}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/IPathHandler.java b/common/source/java/ch/systemsx/cisd/common/utilities/IPathHandler.java
index 3380e2f31d489474eb6c0f791bbcb55725274bde..eada9474a2befeae599a3b7913a3676410314b7a 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/IPathHandler.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/IPathHandler.java
@@ -25,7 +25,7 @@ import java.io.File;
* @see IStoreHandler
* @author Bernd Rinn
*/
-public interface IPathHandler
+public interface IPathHandler extends IStopSignaler
{
/**
* Handles the <var>path</var>. Successful handling is indicated by <var>path</var> being gone
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/IStopSignaler.java b/common/source/java/ch/systemsx/cisd/common/utilities/IStopSignaler.java
new file mode 100644
index 0000000000000000000000000000000000000000..6563a0b8e20a64a57eb0567269b09b09f6472963
--- /dev/null
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/IStopSignaler.java
@@ -0,0 +1,36 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.utilities;
+
+/**
+ * A role that allows to signal that a worker was stopped.
+ *
+ * @author Bernd Rinn
+ */
+public interface IStopSignaler
+{
+
+ /**
+ * Returns <code>true</code>, if the signaler has been stopped and <code>false</code> otherwise.
+ * <p>
+ * Not throwing a StopException, but signaling back that the worker was stopped by means of
+ * this method implies that it was prepared to handle the StopException and clean up
+ * after itself.
+ */
+ boolean isStopped();
+
+}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/IStoreHandler.java b/common/source/java/ch/systemsx/cisd/common/utilities/IStoreHandler.java
index 0ff94bca6176c7ab7ba3abc252a523e982462ce2..34fa6c35087c5d3581aa0d1ee015ea2349aacc16 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/IStoreHandler.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/IStoreHandler.java
@@ -16,23 +16,21 @@
package ch.systemsx.cisd.common.utilities;
-import java.io.File;
-
/**
* Handles items in the file store.
* <p>
* Note that this interface is a higher abstraction of {@link IPathHandler} which works with
- * {@link File}.
+ * {@link java.io.File}.
* </p>
*
* @see IPathHandler
* @author Tomasz Pylak
*/
-public interface IStoreHandler
+public interface IStoreHandler extends IStopSignaler
{
/**
- * Handles given <var>item</var>. Successful handling is indicated by <var>item</var> being
- * gone when the method returns.
+ * Handles given <var>item</var>. Successful handling is indicated by <var>item</var> being gone
+ * when the method returns.
*/
void handle(StoreItem item);
}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/PathHandlerAdapter.java b/common/source/java/ch/systemsx/cisd/common/utilities/PathHandlerAdapter.java
index 43e0a20ce9d5492b94f8ea8fd45bfb2e63dd4b2d..808a8c5dabc4a722b1bd606778a21658fbfee604 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/PathHandlerAdapter.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/PathHandlerAdapter.java
@@ -54,4 +54,9 @@ public class PathHandlerAdapter implements IStoreHandler
pathHandler.handle(asFile(item));
}
+ public boolean isStopped()
+ {
+ return pathHandler.isStopped();
+ }
+
}
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/RemoteDirectoryCopyActivitySensor.java b/common/source/java/ch/systemsx/cisd/common/utilities/RemoteDirectoryCopyActivitySensor.java
index 6cf609f25445c89e805ca9ee8c9cfe831b0ec6e8..6a702dc3f78903b4076e817e53f652d0ecae1cf6 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/RemoteDirectoryCopyActivitySensor.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/RemoteDirectoryCopyActivitySensor.java
@@ -20,14 +20,14 @@ import java.io.File;
import org.apache.log4j.Logger;
-import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IActivitySensor;
+import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IDescribingActivitySensor;
import ch.systemsx.cisd.common.exceptions.StatusWithResult;
import ch.systemsx.cisd.common.exceptions.UnknownLastChangedException;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
/**
- * A {@link IActivitySensor} that senses changes in copy operations to a directory.
+ * A {@link IDescribingActivitySensor} that senses changes in copy operations to a directory.
*
* @author Bernd Rinn
*/
diff --git a/common/source/java/ch/systemsx/cisd/common/utilities/RsyncBasedRecursiveHardLinkMaker.java b/common/source/java/ch/systemsx/cisd/common/utilities/RsyncBasedRecursiveHardLinkMaker.java
index 5ae3bcb668394978c15fa46a2c921bed4d09d5af..bdf367fa33987ddea1b8e0604640c5cd57e0a779 100644
--- a/common/source/java/ch/systemsx/cisd/common/utilities/RsyncBasedRecursiveHardLinkMaker.java
+++ b/common/source/java/ch/systemsx/cisd/common/utilities/RsyncBasedRecursiveHardLinkMaker.java
@@ -20,6 +20,7 @@ import java.io.File;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.TimingParameters;
import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities;
import ch.systemsx.cisd.common.concurrent.InactivityMonitor;
import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IInactivityObserver;
@@ -40,24 +41,12 @@ public class RsyncBasedRecursiveHardLinkMaker implements IDirectoryImmutableCopi
private static final Logger machineLog =
LogFactory.getLogger(LogCategory.MACHINE, RsyncBasedRecursiveHardLinkMaker.class);
- private final static long MILLIS = 1000L;
-
- private final static long DEFAULT_INACTIVITY_TRESHOLD_MILLIS = 300 * MILLIS;
-
private final static int DEFAULT_MAX_ERRORS_TO_IGNORE = 3;
- private final static int DEFAULT_MAX_ATTEMPTS = 10;
-
- private final static long DEFAULT_TIME_TO_SLEEP_AFTER_COPY_FAILS = 5 * MILLIS;
-
- private final long inactivityThresholdMillis;
+ private final TimingParameters timingParameters;
private final int maxErrorsToIgnore;
- private final int maxAttempts;
-
- private final long timeToSleepAfterCopyFails;
-
private final RsyncCopier rsyncCopier;
public interface ILastChangedChecker
@@ -67,20 +56,17 @@ public class RsyncBasedRecursiveHardLinkMaker implements IDirectoryImmutableCopi
public RsyncBasedRecursiveHardLinkMaker()
{
- this(null, DEFAULT_INACTIVITY_TRESHOLD_MILLIS, DEFAULT_MAX_ERRORS_TO_IGNORE,
- DEFAULT_MAX_ATTEMPTS, DEFAULT_TIME_TO_SLEEP_AFTER_COPY_FAILS);
+ this(null, TimingParameters.getDefaultParameters(), DEFAULT_MAX_ERRORS_TO_IGNORE);
}
public RsyncBasedRecursiveHardLinkMaker(File rsyncExecutableOrNull)
{
- this(rsyncExecutableOrNull, DEFAULT_INACTIVITY_TRESHOLD_MILLIS,
- DEFAULT_MAX_ERRORS_TO_IGNORE, DEFAULT_MAX_ATTEMPTS,
- DEFAULT_TIME_TO_SLEEP_AFTER_COPY_FAILS);
+ this(rsyncExecutableOrNull, TimingParameters.getDefaultParameters(),
+ DEFAULT_MAX_ERRORS_TO_IGNORE);
}
public RsyncBasedRecursiveHardLinkMaker(File rsyncExecutableOrNull,
- long inactivityThresholdMillis, int maxErrorsToIgnore, int maxAttempts,
- long timeToSleepAfterCopyFails)
+ TimingParameters timingParameters, int maxErrorsToIgnore)
{
if (rsyncExecutableOrNull == null)
{
@@ -89,12 +75,14 @@ public class RsyncBasedRecursiveHardLinkMaker implements IDirectoryImmutableCopi
{
rsyncCopier = new RsyncCopier(rsyncExecutableOrNull);
}
- this.inactivityThresholdMillis = inactivityThresholdMillis;
- this.timeToSleepAfterCopyFails = timeToSleepAfterCopyFails;
+ this.timingParameters = timingParameters;
this.maxErrorsToIgnore = maxErrorsToIgnore;
- this.maxAttempts = maxAttempts;
}
+ //
+ // IDirectoryImmutableCopier
+ //
+
public boolean copyDirectoryImmutably(final File sourceDirectory,
final File destinationDirectory, final String targetNameOrNull)
{
@@ -110,19 +98,19 @@ public class RsyncBasedRecursiveHardLinkMaker implements IDirectoryImmutableCopi
}
};
boolean ok;
- int attempt = 0;
+ int counter = 0;
while (true)
{
ok =
createHardLinks(sourceDirectory, destinationDirectory, targetNameOrNull,
target, observer);
- if (ok || attempt++ > maxAttempts)
+ if (ok || counter++ > timingParameters.getMaxRetriesOnFailure())
{
break;
}
if (ok == false)
{
- ConcurrencyUtilities.sleep(timeToSleepAfterCopyFails);
+ ConcurrencyUtilities.sleep(timingParameters.getIntervalToWaitAfterFailureMillis());
}
}
return ok;
@@ -133,7 +121,7 @@ public class RsyncBasedRecursiveHardLinkMaker implements IDirectoryImmutableCopi
{
final InactivityMonitor monitor =
new InactivityMonitor(new RemoteDirectoryCopyActivitySensor(target,
- maxErrorsToIgnore), observer, inactivityThresholdMillis, true);
+ maxErrorsToIgnore), observer, timingParameters.getTimeoutMillis(), true);
final boolean result =
rsyncCopier.copyDirectoryImmutably(sourceDirectory, destinationDirectory,
targetNameOrNull);
diff --git a/common/source/java/org/apache/commons/io/FileCopyUtils.java b/common/source/java/org/apache/commons/io/FileCopyUtils.java
new file mode 100644
index 0000000000000000000000000000000000000000..5b3594e2da6a549c246c593d24da1fd315fe3e33
--- /dev/null
+++ b/common/source/java/org/apache/commons/io/FileCopyUtils.java
@@ -0,0 +1,562 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.commons.io;
+
+import java.io.File;
+import java.io.FileFilter;
+import java.io.FileInputStream;
+import java.io.FileNotFoundException;
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import ch.systemsx.cisd.common.concurrent.IActivityObserver;
+
+/**
+ * General file copying utilities.
+ * <p>
+ * Origin of code: Excalibur, Alexandria, Commons-Utils, CISD common
+ *
+ * @author <a href="mailto:burton@relativity.yi.org">Kevin A. Burton</A>
+ * @author <a href="mailto:sanders@apache.org">Scott Sanders</a>
+ * @author <a href="mailto:dlr@finemaltcoding.com">Daniel Rall</a>
+ * @author <a href="mailto:Christoph.Reck@dlr.de">Christoph.Reck</a>
+ * @author <a href="mailto:peter@apache.org">Peter Donald</a>
+ * @author <a href="mailto:jefft@apache.org">Jeff Turner</a>
+ * @author Matthew Hawthorne
+ * @author <a href="mailto:jeremias@apache.org">Jeremias Maerki</a>
+ * @author Stephen Colebourne
+ * @author Ian Springer
+ * @author Chris Eldredge
+ * @author Jim Harrington
+ * @author Niall Pemberton
+ * @author Sandy McArthur
+ * @author Bernd Rinn
+ */
+public class FileCopyUtils
+{
+
+ private FileCopyUtils()
+ {
+ // Not to be instantiated.
+ }
+
+ /**
+ * Copies a file or directory to a new location, preserving the file date.
+ * <p>
+ * This method copies the contents of the specified source file to the specified destination
+ * file. The directory holding the destination file is created if it does not exist. If the
+ * destination file exists, then this method will overwrite it.
+ *
+ * @param source an existing file to copy, must not be <code>null</code>
+ * @param destination the new file, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ */
+ public static void copy(File source, File destination) throws IOException
+ {
+ if (source.isDirectory())
+ {
+ copyDirectory(source, destination);
+ } else
+ {
+ copyFile(source, destination);
+ }
+ }
+
+ /**
+ * Copies a file or directory to a new location, preserving the file date.
+ * <p>
+ * This method copies the contents of the specified source file to the specified destination
+ * file. The directory holding the destination file is created if it does not exist. If the
+ * destination file exists, then this method will overwrite it.
+ *
+ * @param source an existing file to copy, must not be <code>null</code>
+ * @param destination the new file, must not be <code>null</code>
+ * @param observerOrNull activity observer of the copy process
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ */
+ public static void copy(File source, File destination, IActivityObserver observerOrNull)
+ throws IOException
+ {
+ if (source.isDirectory())
+ {
+ copyDirectory(source, destination, observerOrNull);
+ } else
+ {
+ copyFile(source, destination);
+ }
+ }
+
+ /**
+ * Copies a file or directory to a directory preserving the file or directory date.
+ * <p>
+ * This method copies the contents of the specified source file to a file of the same name in
+ * the specified destination directory. The destination directory is created if it does not
+ * exist. If the destination file exists, then this method will overwrite it.
+ *
+ * @param source an existing file to copy, must not be <code>null</code>
+ * @param destDir the directory to place the copy in, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ */
+ public static void copyToDirectory(File source, File destDir) throws IOException
+ {
+ if (source.isDirectory())
+ {
+ copyDirectoryToDirectory(source, destDir);
+ } else
+ {
+ copyFileToDirectory(source, destDir);
+ }
+ }
+
+ /**
+ * Copies a file or directory to a directory preserving the file or directory date.
+ * <p>
+ * This method copies the contents of the specified source file to a file of the same name in
+ * the specified destination directory. The destination directory is created if it does not
+ * exist. If the destination file exists, then this method will overwrite it.
+ *
+ * @param source an existing file to copy, must not be <code>null</code>
+ * @param destination the directory to place the copy in, must not be <code>null</code>
+ * @param observerOrNull activity observer of the copy process
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ */
+ public static void copyToDirectory(File source, File destination,
+ IActivityObserver observerOrNull) throws IOException
+ {
+ if (source.isDirectory())
+ {
+ copyDirectoryToDirectory(source, destination, observerOrNull);
+ } else
+ {
+ copyFileToDirectory(source, destination);
+ }
+ }
+
+ /**
+ * Copies a file to a directory preserving the file date.
+ * <p>
+ * This method copies the contents of the specified source file to a file of the same name in
+ * the specified destination directory. The destination directory is created if it does not
+ * exist. If the destination file exists, then this method will overwrite it.
+ *
+ * @param srcFile an existing file to copy, must not be <code>null</code>
+ * @param destDir the directory to place the copy in, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ * @since Commons IO 1.3
+ */
+ public static void copyFileToDirectory(File srcFile, File destDir) throws IOException
+ {
+ if (destDir == null)
+ {
+ throw new NullPointerException("Destination must not be null");
+ }
+ if (destDir.exists() && destDir.isDirectory() == false)
+ {
+ throw new IllegalArgumentException("Destination '" + destDir + "' is not a directory");
+ }
+ copyFile(srcFile, new File(destDir, srcFile.getName()));
+ }
+
+ /**
+ * Copies a file to a new location, preserving the file date.
+ * <p>
+ * This method copies the contents of the specified source file to the specified destination
+ * file. The directory holding the destination file is created if it does not exist. If the
+ * destination file exists, then this method will overwrite it.
+ *
+ * @param srcFile an existing file to copy, must not be <code>null</code>
+ * @param destFile the new file, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ */
+ public static void copyFile(File srcFile, File destFile) throws IOException
+ {
+ if (srcFile == null)
+ {
+ throw new NullPointerException("Source must not be null");
+ }
+ if (destFile == null)
+ {
+ throw new NullPointerException("Destination must not be null");
+ }
+ if (srcFile.exists() == false)
+ {
+ throw new FileNotFoundException("Source '" + srcFile + "' does not exist");
+ }
+ if (srcFile.isDirectory())
+ {
+ throw new IOException("Source '" + srcFile + "' exists but is a directory");
+ }
+ if (srcFile.getCanonicalPath().equals(destFile.getCanonicalPath()))
+ {
+ throw new IOException("Source '" + srcFile + "' and destination '" + destFile
+ + "' are the same");
+ }
+ if (destFile.getParentFile() != null && destFile.getParentFile().exists() == false)
+ {
+ if (destFile.getParentFile().mkdirs() == false)
+ {
+ throw new IOException("Destination '" + destFile + "' directory cannot be created");
+ }
+ }
+ if (destFile.exists() && destFile.canWrite() == false)
+ {
+ throw new IOException("Destination '" + destFile + "' exists but is read-only");
+ }
+ doCopyFile(srcFile, destFile);
+ }
+
+ /**
+ * Internal copy file method.
+ *
+ * @param srcFile the validated source file, must not be <code>null</code>
+ * @param destFile the validated destination file, must not be <code>null</code>
+ * @throws IOException if an error occurs
+ */
+ private static void doCopyFile(File srcFile, File destFile) throws IOException
+ {
+ if (destFile.exists() && destFile.isDirectory())
+ {
+ throw new IOException("Destination '" + destFile + "' exists but is a directory");
+ }
+
+ FileInputStream input = new FileInputStream(srcFile);
+ try
+ {
+ FileOutputStream output = new FileOutputStream(destFile);
+ try
+ {
+ IOUtils.copy(input, output);
+ } finally
+ {
+ IOUtils.closeQuietly(output);
+ }
+ } finally
+ {
+ IOUtils.closeQuietly(input);
+ }
+
+ if (srcFile.length() != destFile.length())
+ {
+ throw new IOException("Failed to copy full contents from '" + srcFile + "' to '"
+ + destFile + "'");
+ }
+ destFile.setLastModified(srcFile.lastModified());
+ }
+
+ /**
+ * Copies a directory to within another directory preserving the file dates.
+ * <p>
+ * This method copies the source directory and all its contents to a directory of the same name
+ * in the specified destination directory.
+ * <p>
+ * The destination directory is created if it does not exist. If the destination directory did
+ * exist, then this method merges the source with the destination, with the source taking
+ * precedence.
+ *
+ * @param srcDir an existing directory to copy, must not be <code>null</code>
+ * @param destDir the directory to place the copy in, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ * @since Commons IO 1.2
+ */
+ public static void copyDirectoryToDirectory(File srcDir, File destDir) throws IOException
+ {
+ copyDirectoryToDirectory(srcDir, destDir, null);
+ }
+
+ /**
+ * Copies a directory to within another directory preserving the file dates.
+ * <p>
+ * This method copies the source directory and all its contents to a directory of the same name
+ * in the specified destination directory.
+ * <p>
+ * The destination directory is created if it does not exist. If the destination directory did
+ * exist, then this method merges the source with the destination, with the source taking
+ * precedence.
+ *
+ * @param srcDir an existing directory to copy, must not be <code>null</code>
+ * @param destDir the directory to place the copy in, must not be <code>null</code>
+ * @param observerOrNull activity observer of the copy process
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ * @since Commons IO 1.2
+ */
+ public static void copyDirectoryToDirectory(File srcDir, File destDir,
+ IActivityObserver observerOrNull) throws IOException
+ {
+ if (srcDir == null)
+ {
+ throw new NullPointerException("Source must not be null");
+ }
+ if (srcDir.exists() && srcDir.isDirectory() == false)
+ {
+ throw new IllegalArgumentException("Source '" + destDir + "' is not a directory");
+ }
+ if (destDir == null)
+ {
+ throw new NullPointerException("Destination must not be null");
+ }
+ if (destDir.exists() && destDir.isDirectory() == false)
+ {
+ throw new IllegalArgumentException("Destination '" + destDir + "' is not a directory");
+ }
+ copyDirectory(srcDir, new File(destDir, srcDir.getName()), null, observerOrNull);
+ }
+
+ /**
+ * Copies a whole directory to a new location preserving the file dates.
+ * <p>
+ * This method copies the specified directory and all its child directories and files to the
+ * specified destination. The destination is the new location and name of the directory.
+ * <p>
+ * The destination directory is created if it does not exist. If the destination directory did
+ * exist, then this method merges the source with the destination, with the source taking
+ * precedence.
+ *
+ * @param srcDir an existing directory to copy, must not be <code>null</code>
+ * @param destDir the new directory, must not be <code>null</code>
+ * @param observerOrNull activity observer of the copy process
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ * @since Commons IO 1.1
+ */
+ public static void copyDirectory(File srcDir, File destDir, IActivityObserver observerOrNull)
+ throws IOException
+ {
+ copyDirectory(srcDir, destDir, null, observerOrNull);
+ }
+
+ /**
+ * Copies a whole directory to a new location.
+ * <p>
+ * This method copies the contents of the specified source directory to within the specified
+ * destination directory.
+ * <p>
+ * The destination directory is created if it does not exist. If the destination directory did
+ * exist, then this method merges the source with the destination, with the source taking
+ * precedence.
+ *
+ * @param srcDir an existing directory to copy, must not be <code>null</code>
+ * @param destDir the new directory, must not be <code>null</code>
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ * @since Commons IO 1.1
+ */
+ public static void copyDirectory(File srcDir, File destDir) throws IOException
+ {
+ copyDirectory(srcDir, destDir, (FileFilter) null);
+ }
+
+ /**
+ * Copies a filtered directory to a new location.
+ * <p>
+ * This method copies the contents of the specified source directory to within the specified
+ * destination directory.
+ * <p>
+ * The destination directory is created if it does not exist. If the destination directory did
+ * exist, then this method merges the source with the destination, with the source taking
+ * precedence.
+ * <h4>Example: Copy directories only</h4>
+ *
+ * <pre>
+ * // only copy the directory structure
+ * FileUtils.copyDirectory(srcDir, destDir, DirectoryFileFilter.DIRECTORY, false);
+ * </pre>
+ *
+ * <h4>Example: Copy directories and txt files</h4>
+ *
+ * <pre>
+ * // Create a filter for ".txt" files
+ * IOFileFilter txtSuffixFilter = FileFilterUtils.suffixFileFilter(".txt");
+ * IOFileFilter txtFiles = FileFilterUtils.andFileFilter(FileFileFilter.FILE, txtSuffixFilter);
+ * // Create a filter for either directories or ".txt" files
+ * FileFilter filter = FileFilterUtils.orFileFilter(DirectoryFileFilter.DIRECTORY, txtFiles);
+ * // Copy using the filter
+ * FileUtils.copyDirectory(srcDir, destDir, filter, false);
+ * </pre>
+ *
+ * @param srcDir an existing directory to copy, must not be <code>null</code>
+ * @param destDir the new directory, must not be <code>null</code>
+ * @param filter the filter to apply, null means copy all directories and files
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ * @since Commons IO 1.4
+ */
+ public static void copyDirectory(File srcDir, File destDir, FileFilter filter)
+ throws IOException
+ {
+ copyDirectory(srcDir, destDir, filter, null);
+ }
+
+ /**
+ * Copies a filtered directory to a new location.
+ * <p>
+ * This method copies the contents of the specified source directory to within the specified
+ * destination directory.
+ * <p>
+ * The destination directory is created if it does not exist. If the destination directory did
+ * exist, then this method merges the source with the destination, with the source taking
+ * precedence.
+ * <h4>Example: Copy directories only</h4>
+ *
+ * <pre>
+ * // only copy the directory structure
+ * FileUtils.copyDirectory(srcDir, destDir, DirectoryFileFilter.DIRECTORY, false);
+ * </pre>
+ *
+ * <h4>Example: Copy directories and txt files</h4>
+ *
+ * <pre>
+ * // Create a filter for ".txt" files
+ * IOFileFilter txtSuffixFilter = FileFilterUtils.suffixFileFilter(".txt");
+ * IOFileFilter txtFiles = FileFilterUtils.andFileFilter(FileFileFilter.FILE, txtSuffixFilter);
+ * // Create a filter for either directories or ".txt" files
+ * FileFilter filter = FileFilterUtils.orFileFilter(DirectoryFileFilter.DIRECTORY, txtFiles);
+ * // Copy using the filter
+ * FileUtils.copyDirectory(srcDir, destDir, filter, false);
+ * </pre>
+ *
+ * @param srcDir an existing directory to copy, must not be <code>null</code>
+ * @param destDir the new directory, must not be <code>null</code>
+ * @param filter the filter to apply, null means copy all directories and files
+ * @param observerOrNull activity observer of the copy process
+ * @throws NullPointerException if source or destination is <code>null</code>
+ * @throws IOException if source or destination is invalid
+ * @throws IOException if an IO error occurs during copying
+ * @since Commons IO 1.4
+ */
+ public static void copyDirectory(File srcDir, File destDir, FileFilter filter,
+ IActivityObserver observerOrNull) throws IOException
+ {
+ if (srcDir == null)
+ {
+ throw new NullPointerException("Source must not be null");
+ }
+ if (destDir == null)
+ {
+ throw new NullPointerException("Destination must not be null");
+ }
+ if (srcDir.exists() == false)
+ {
+ throw new FileNotFoundException("Source '" + srcDir + "' does not exist");
+ }
+ if (srcDir.isDirectory() == false)
+ {
+ throw new IOException("Source '" + srcDir + "' exists but is not a directory");
+ }
+ if (srcDir.getCanonicalPath().equals(destDir.getCanonicalPath()))
+ {
+ throw new IOException("Source '" + srcDir + "' and destination '" + destDir
+ + "' are the same");
+ }
+
+ // Cater for destination being directory within the source directory (see IO-141)
+ List<String> exclusionList = null;
+ if (destDir.getCanonicalPath().startsWith(srcDir.getCanonicalPath()))
+ {
+ File[] srcFiles = filter == null ? srcDir.listFiles() : srcDir.listFiles(filter);
+ if (srcFiles != null && srcFiles.length > 0)
+ {
+ exclusionList = new ArrayList<String>(srcFiles.length);
+ for (int i = 0; i < srcFiles.length; i++)
+ {
+ File copiedFile = new File(destDir, srcFiles[i].getName());
+ exclusionList.add(copiedFile.getCanonicalPath());
+ }
+ }
+ }
+ doCopyDirectory(srcDir, destDir, filter, exclusionList, observerOrNull);
+ }
+
+ /**
+ * Internal copy directory method.
+ *
+ * @param srcDir the validated source directory, must not be <code>null</code>
+ * @param destDir the validated destination directory, must not be <code>null</code>
+ * @param filter the filter to apply, null means copy all directories and files
+ * @param exclusionList List of files and directories to exclude from the copy, may be null
+ * @param observerOrNull activity observer of the copy process
+ * @throws IOException if an error occurs
+ * @since Commons IO 1.1
+ */
+ private static void doCopyDirectory(File srcDir, File destDir, FileFilter filter,
+ List<String> exclusionList, IActivityObserver observerOrNull) throws IOException
+ {
+ if (destDir.exists())
+ {
+ if (destDir.isDirectory() == false)
+ {
+ throw new IOException("Destination '" + destDir + "' exists but is not a directory");
+ }
+ } else
+ {
+ if (destDir.mkdirs() == false)
+ {
+ throw new IOException("Destination '" + destDir + "' directory cannot be created");
+ }
+ destDir.setLastModified(srcDir.lastModified());
+ }
+ if (destDir.canWrite() == false)
+ {
+ throw new IOException("Destination '" + destDir + "' cannot be written to");
+ }
+ // recurse
+ File[] files = filter == null ? srcDir.listFiles() : srcDir.listFiles(filter);
+ if (files == null)
+ { // null if security restricted
+ throw new IOException("Failed to list contents of " + srcDir);
+ }
+ for (int i = 0; i < files.length; i++)
+ {
+ if (observerOrNull != null)
+ {
+ observerOrNull.update();
+ }
+ File copiedFile = new File(destDir, files[i].getName());
+ if (exclusionList == null || !exclusionList.contains(files[i].getCanonicalPath()))
+ {
+ if (files[i].isDirectory())
+ {
+ doCopyDirectory(files[i], copiedFile, filter, exclusionList, observerOrNull);
+ } else
+ {
+ doCopyFile(files[i], copiedFile);
+ }
+ }
+ }
+ }
+}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/TimingParametersTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/TimingParametersTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..86dc893c885a893c651c34644eacc9be5f412b86
--- /dev/null
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/TimingParametersTest.java
@@ -0,0 +1,89 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common;
+
+import java.util.Properties;
+
+import org.apache.commons.lang.time.DateUtils;
+import org.testng.annotations.Test;
+
+import static org.testng.AssertJUnit.*;
+
+/**
+ * Test cases for the {@link TimingParameters}.
+ *
+ * @author Bernd Rinn
+ */
+public class TimingParametersTest
+{
+
+ @Test
+ public void testCreateTimingParametersDefaults()
+ {
+ final Properties props = new Properties();
+ final TimingParameters parameters = TimingParameters.create(props);
+ assertEquals(TimingParameters.getDefaultParameters(), parameters);
+ }
+
+ @Test
+ public void testCreateTimingParameters()
+ {
+ final int timeout = 33;
+ final int maxRetries = 7;
+ final int failureInterval = 13;
+ final Properties props = new Properties();
+ props.put(TimingParameters.TIMEOUT_PROPERTY, Integer.toString(timeout));
+ props.put(TimingParameters.MAX_RETRY_PROPERTY, Integer.toString(maxRetries));
+ props.put(TimingParameters.FAILURE_INTERVAL, Integer.toString(failureInterval));
+ final TimingParameters parameters = TimingParameters.create(props);
+ assertEquals(timeout * DateUtils.MILLIS_PER_SECOND, parameters.getTimeoutMillis());
+ assertEquals(maxRetries, parameters.getMaxRetriesOnFailure());
+ assertEquals(failureInterval * DateUtils.MILLIS_PER_SECOND, parameters
+ .getIntervalToWaitAfterFailureMillis());
+ }
+
+ @Test
+ public void testHasTimingParametersFalse()
+ {
+ final Properties props = new Properties();
+ assertFalse(TimingParameters.hasTimingParameters(props));
+ }
+
+ @Test
+ public void testHasTimingParametersTrue()
+ {
+ final Properties props = new Properties();
+ props.put(TimingParameters.TIMEOUT_PROPERTY, "30");
+ assertTrue(TimingParameters.hasTimingParameters(props));
+ }
+
+ @Test
+ public void testHasTimingParametersTrue2()
+ {
+ final Properties props = new Properties();
+ props.put(TimingParameters.MAX_RETRY_PROPERTY, "2");
+ assertTrue(TimingParameters.hasTimingParameters(props));
+ }
+
+ @Test
+ public void testHasTimingParametersTrue3()
+ {
+ final Properties props = new Properties();
+ props.put(TimingParameters.FAILURE_INTERVAL, "xxx");
+ assertTrue(TimingParameters.hasTimingParameters(props));
+ }
+}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/ConcurrencyUtilitiesTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/ConcurrencyUtilitiesTest.java
index 38c1f7538921f3b5643c85541f69b35e3614593c..a2f4553bcd54f0c525e25c6f103089f36b907f18 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/ConcurrencyUtilitiesTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/ConcurrencyUtilitiesTest.java
@@ -18,18 +18,25 @@ package ch.systemsx.cisd.common.concurrent;
import static org.testng.AssertJUnit.*;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Timer;
import java.util.TimerTask;
import java.util.concurrent.Callable;
import java.util.concurrent.Future;
import java.util.concurrent.ThreadPoolExecutor;
-import org.testng.annotations.BeforeClass;
+import org.testng.annotations.AfterClass;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
import org.testng.annotations.Test;
+import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities.ILogSettings;
import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.logging.LogLevel;
/**
* Test cases for {@link ConcurrencyUtilities}.
@@ -41,12 +48,85 @@ public class ConcurrencyUtilitiesTest
private final static String name = "This is the pool name";
- @BeforeClass
+ private static class LogRecord
+ {
+ final LogLevel level;
+
+ final String message;
+
+ LogRecord(LogLevel level, String message)
+ {
+ this.level = level;
+ this.message = message;
+ }
+ }
+
+ private class AssertingLogger implements ISimpleLogger
+ {
+ private final List<LogRecord> records = new ArrayList<LogRecord>();
+
+ public void log(LogLevel level, String message)
+ {
+ records.add(new LogRecord(level, message));
+ }
+
+ public void assertNumberOfMessage(int expectedNumberOfMessages)
+ {
+ assertEquals(expectedNumberOfMessages, records.size());
+ }
+
+ public void assertEq(int i, LogLevel expectedLevel, String expectedMessage)
+ {
+ assertEquals(expectedLevel, records.get(i).level);
+ assertEquals(expectedMessage, records.get(i).message);
+ }
+
+ }
+
+ private ILogSettings logSettings;
+
+ private AssertingLogger logger;
+
+ @BeforeTest
public void init()
{
LogInitializer.init();
}
+ @BeforeMethod
+ public void beforeMethod()
+ {
+ logger = new AssertingLogger();
+ createLogSettings(LogLevel.WARN);
+ }
+
+ @AfterClass
+ public void clearThreadInterruptionState()
+ {
+ Thread.interrupted();
+ }
+
+ private void createLogSettings(final LogLevel level)
+ {
+ logSettings = new ILogSettings()
+ {
+ public LogLevel getLogLevelForError()
+ {
+ return level;
+ }
+
+ public AssertingLogger getLogger()
+ {
+ return logger;
+ }
+
+ public String getOperationName()
+ {
+ return name;
+ }
+ };
+ }
+
@Test
public void testTryGetFutureOK()
{
@@ -60,13 +140,37 @@ public class ConcurrencyUtilitiesTest
return valueProvided;
}
});
- final String valueObtained = ConcurrencyUtilities.tryGetResult(future, 200L);
+ final String valueObtained =
+ ConcurrencyUtilities.tryGetResult(future, 200L, logSettings, true);
assertEquals(valueProvided, valueObtained);
assertTrue(future.isDone());
+ logger.assertNumberOfMessage(0);
}
@Test
public void testGetExecutionResultOK()
+ {
+ final String valueProvided = "This is the execution return value";
+ final ThreadPoolExecutor eservice =
+ new NamingThreadPoolExecutor(name).corePoolSize(1).maximumPoolSize(2);
+ final Future<String> future = eservice.submit(new Callable<String>()
+ {
+ public String call() throws Exception
+ {
+ return valueProvided;
+ }
+ });
+ final ExecutionResult<String> result =
+ ConcurrencyUtilities.getResult(future, 200L, logSettings);
+ assertEquals(ExecutionStatus.COMPLETE, result.getStatus());
+ assertNull(result.tryGetException());
+ assertEquals(valueProvided, result.tryGetResult());
+ assertTrue(future.isDone());
+ logger.assertNumberOfMessage(0);
+ }
+
+ @Test
+ public void testGetExecutionResultOKWithoutLogging()
{
final String valueProvided = "This is the execution return value";
final ThreadPoolExecutor eservice =
@@ -104,9 +208,12 @@ public class ConcurrencyUtilitiesTest
return null;
}
});
- final String shouldBeNull = ConcurrencyUtilities.tryGetResult(future, 20L);
+ final String shouldBeNull =
+ ConcurrencyUtilities.tryGetResult(future, 20L, logSettings, true);
assertNull(shouldBeNull);
assertTrue(future.isDone());
+ logger.assertNumberOfMessage(1);
+ logger.assertEq(0, LogLevel.WARN, name + ": timeout of 0.02 s exceeded, cancelled.");
}
@Test(groups = "slow")
@@ -128,17 +235,67 @@ public class ConcurrencyUtilitiesTest
return null;
}
});
- final ExecutionResult<String> result = ConcurrencyUtilities.getResult(future, 20L);
+ final ExecutionResult<String> result =
+ ConcurrencyUtilities.getResult(future, 20L, logSettings);
assertEquals(ExecutionStatus.TIMED_OUT, result.getStatus());
assertNull(result.tryGetResult());
assertNull(result.tryGetException());
assertTrue(future.isDone());
assertTrue(future.isCancelled());
+ logger.assertNumberOfMessage(1);
+ logger.assertEq(0, LogLevel.WARN, name + ": timeout of 0.02 s exceeded, cancelled.");
+ }
+
+ @Test(groups = "slow")
+ public void testGetExecutionResultNoTimeoutDueToSensor()
+ {
+ final RecordingActivityObserverSensor sensor = new RecordingActivityObserverSensor();
+ final ThreadPoolExecutor eservice =
+ new NamingThreadPoolExecutor(name).corePoolSize(1).maximumPoolSize(2);
+ final Timer updatingTimer = new Timer();
+ final String msg = "success";
+ try
+ {
+ updatingTimer.schedule(new TimerTask()
+ {
+ @Override
+ public void run()
+ {
+ sensor.update();
+ }
+ }, 0L, 10L);
+ final Future<String> future = eservice.submit(new Callable<String>()
+ {
+ public String call() throws Exception
+ {
+ try
+ {
+ Thread.sleep(200L);
+ } catch (InterruptedException ex)
+ {
+ throw new CheckedExceptionTunnel(ex);
+ }
+ return msg;
+ }
+ });
+ final ExecutionResult<String> result =
+ ConcurrencyUtilities.getResult(future, 20L, true, logSettings, sensor);
+ assertEquals(ExecutionStatus.COMPLETE, result.getStatus());
+ assertEquals(msg, result.tryGetResult());
+ assertNull(result.tryGetException());
+ assertTrue(future.isDone());
+ assertFalse(future.isCancelled());
+ logger.assertNumberOfMessage(0);
+ } finally
+ {
+ updatingTimer.cancel();
+ }
}
@Test(groups = "slow")
public void testGetExecutionResultTimeoutWithoutCancelation()
{
+ createLogSettings(LogLevel.INFO);
final ThreadPoolExecutor eservice =
new NamingThreadPoolExecutor(name).corePoolSize(1).maximumPoolSize(2);
final Future<String> future = eservice.submit(new Callable<String>()
@@ -156,12 +313,14 @@ public class ConcurrencyUtilitiesTest
}
});
final ExecutionResult<String> result =
- ConcurrencyUtilities.getResult(future, 20L, false, null, null);
+ ConcurrencyUtilities.getResult(future, 20L, false, logSettings);
assertEquals(ExecutionStatus.TIMED_OUT, result.getStatus());
assertNull(result.tryGetResult());
assertNull(result.tryGetException());
assertFalse(future.isDone());
assertFalse(future.isCancelled());
+ logger.assertNumberOfMessage(1);
+ logger.assertEq(0, LogLevel.INFO, name + ": timeout of 0.02 s exceeded.");
}
@Test
@@ -193,11 +352,14 @@ public class ConcurrencyUtilitiesTest
thread.interrupt();
}
}, 20L);
- final String shouldBeNull = ConcurrencyUtilities.tryGetResult(future, 200L, false);
+ final String shouldBeNull =
+ ConcurrencyUtilities.tryGetResult(future, 200L, logSettings, false);
t.cancel();
assertNull(shouldBeNull);
assertTrue(future.isCancelled());
assertFalse(Thread.interrupted());
+ logger.assertNumberOfMessage(1);
+ logger.assertEq(0, LogLevel.WARN, name + ": interrupted.");
}
@Test(expectedExceptions =
@@ -237,6 +399,7 @@ public class ConcurrencyUtilitiesTest
@Test
public void testGetExecutionResultInterrupted()
{
+ createLogSettings(LogLevel.DEBUG);
final ThreadPoolExecutor eservice =
new NamingThreadPoolExecutor(name).corePoolSize(1).maximumPoolSize(2);
final Thread thread = Thread.currentThread();
@@ -263,23 +426,36 @@ public class ConcurrencyUtilitiesTest
thread.interrupt();
}
}, 20L);
- final ExecutionResult<String> result = ConcurrencyUtilities.getResult(future, 200L);
+ final ExecutionResult<String> result =
+ ConcurrencyUtilities.getResult(future, 200L, logSettings);
t.cancel();
assertEquals(ExecutionStatus.INTERRUPTED, result.getStatus());
assertNull(result.tryGetResult());
assertNull(result.tryGetException());
assertTrue(future.isCancelled());
assertFalse(Thread.interrupted());
+ logger.assertNumberOfMessage(1);
+ logger.assertEq(0, LogLevel.DEBUG, name + ": interrupted.");
}
private static class TaggedException extends RuntimeException
{
private static final long serialVersionUID = 1L;
+
+ public TaggedException()
+ {
+ }
+
+ public TaggedException(String msg)
+ {
+ super(msg);
+ }
}
@Test
public void testGetExecutionResultException()
{
+ createLogSettings(LogLevel.ERROR);
final ThreadPoolExecutor eservice =
new NamingThreadPoolExecutor(name).corePoolSize(1).maximumPoolSize(2);
final Future<String> future = eservice.submit(new Callable<String>()
@@ -289,23 +465,54 @@ public class ConcurrencyUtilitiesTest
throw new TaggedException();
}
});
- final ExecutionResult<String> result = ConcurrencyUtilities.getResult(future, 100L);
+ final ExecutionResult<String> result =
+ ConcurrencyUtilities.getResult(future, 100L, logSettings);
assertEquals(ExecutionStatus.EXCEPTION, result.getStatus());
assertTrue(result.tryGetException() instanceof TaggedException);
assertNull(result.tryGetResult());
assertTrue(future.isDone());
+ logger.assertNumberOfMessage(1);
+ logger.assertEq(0, LogLevel.ERROR, name + ": exception: <no message> [TaggedException].");
}
@Test
public void testTryGetFutureException()
{
+ final String msg = "This is some sort of error message";
final ThreadPoolExecutor eservice =
new NamingThreadPoolExecutor(name).corePoolSize(1).maximumPoolSize(2);
final Future<String> future = eservice.submit(new Callable<String>()
{
public String call() throws Exception
{
- throw new TaggedException();
+ throw new TaggedException(msg);
+ }
+ });
+ try
+ {
+ ConcurrencyUtilities.tryGetResult(future, 100L, logSettings, true);
+ fail("Should have been a TaggedException");
+ } catch (TaggedException ex)
+ {
+ // Good
+ }
+ assertTrue(future.isDone());
+ logger.assertNumberOfMessage(1);
+ logger.assertEq(0, LogLevel.WARN, name
+ + ": exception: This is some sort of error message [TaggedException].");
+ }
+
+ @Test
+ public void testTryGetFutureExceptionWithoutLogging()
+ {
+ final String msg = "This is some sort of error message";
+ final ThreadPoolExecutor eservice =
+ new NamingThreadPoolExecutor(name).corePoolSize(1).maximumPoolSize(2);
+ final Future<String> future = eservice.submit(new Callable<String>()
+ {
+ public String call() throws Exception
+ {
+ throw new TaggedException(msg);
}
});
try
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/InactivityMonitorTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/InactivityMonitorTest.java
index 7c292e4405499f8a2fd4306ba9781303d9eb2f8a..8ddda4c08e9386ecc3eb483ed1eed130f3c0f5e4 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/InactivityMonitorTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/InactivityMonitorTest.java
@@ -27,7 +27,7 @@ import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
-import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IActivitySensor;
+import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IDescribingActivitySensor;
import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IInactivityObserver;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.test.StoringUncaughtExceptionHandler;
@@ -43,11 +43,11 @@ public class InactivityMonitorTest
private static final long TIME_TO_WAIT_MILLIS = 4 * INACTIVITY_THRESHOLD_MILLIS;
- private final static long DELTA = 10L;
+ private final static long DELTA = 20L;
private Mockery context;
- private IActivitySensor sensor;
+ private IDescribingActivitySensor sensor;
private IInactivityObserver observer;
@@ -130,7 +130,7 @@ public class InactivityMonitorTest
{
exceptionHandler.reset();
context = new Mockery();
- sensor = context.mock(IActivitySensor.class);
+ sensor = context.mock(IDescribingActivitySensor.class);
observer = context.mock(IInactivityObserver.class);
}
@@ -153,7 +153,7 @@ public class InactivityMonitorTest
context.checking(new Expectations()
{
{
- atLeast(1).of(sensor).getTimeOfLastActivityMoreRecentThan(
+ atLeast(1).of(sensor).getLastActivityMillisMoreRecentThan(
INACTIVITY_THRESHOLD_MILLIS);
will(new ReturnNowMinus(0L));
}
@@ -173,7 +173,7 @@ public class InactivityMonitorTest
context.checking(new Expectations()
{
{
- one(sensor).getTimeOfLastActivityMoreRecentThan(INACTIVITY_THRESHOLD_MILLIS);
+ one(sensor).getLastActivityMillisMoreRecentThan(INACTIVITY_THRESHOLD_MILLIS);
will(new ReturnNowMinus(2 * INACTIVITY_THRESHOLD_MILLIS));
one(sensor).describeInactivity(with(new NowMatcher()));
will(returnValue(descriptionOfInactivity));
@@ -190,14 +190,16 @@ public class InactivityMonitorTest
context.assertIsSatisfied();
}
- @Test
+ @Test(groups = "slow")
public void testInactivityMultipleTimes() throws Throwable
{
+ // Wait for system to become quiet to get more accurate measurement.
+ ConcurrencyUtilities.sleep(300L);
final String descriptionOfInactivity = "DESCRIPTION";
context.checking(new Expectations()
{
{
- atLeast(3).of(sensor).getTimeOfLastActivityMoreRecentThan(
+ atLeast(3).of(sensor).getLastActivityMillisMoreRecentThan(
INACTIVITY_THRESHOLD_MILLIS);
will(new ReturnNowMinus(2 * INACTIVITY_THRESHOLD_MILLIS));
atLeast(3).of(sensor).describeInactivity(with(new NowMatcher()));
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/MonitoringProxyTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/MonitoringProxyTest.java
index a37ddde3fa67457b0579897430f5a2cdcc93d63f..d0598c77b076b55f98bb919138d47ee49a48bc36 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/MonitoringProxyTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/concurrent/MonitoringProxyTest.java
@@ -16,18 +16,25 @@
package ch.systemsx.cisd.common.concurrent;
-import static org.testng.AssertJUnit.*;
+import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.assertNull;
+import static org.testng.AssertJUnit.assertTrue;
import java.util.Timer;
import java.util.TimerTask;
import java.util.regex.Pattern;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeTest;
+import org.testng.annotations.BeforeClass;
+import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
+import ch.systemsx.cisd.common.TimingParameters;
import ch.systemsx.cisd.common.exceptions.StopException;
import ch.systemsx.cisd.common.exceptions.TimeoutException;
+import ch.systemsx.cisd.common.logging.ConsoleLogger;
+import ch.systemsx.cisd.common.logging.ISimpleLogger;
/**
* Test cases for the {@link MonitoringProxy}.
@@ -47,19 +54,30 @@ public class MonitoringProxyTest
private static final Status THE_STATUS = Status.TWO;
- private static final long TIMEOUT_MILLIS = 50L;
+ private static final long TIMEOUT_MILLIS = 100L;
private volatile Thread threadToStop;
-
+
+ private RecordingActivityObserverSensor observerSensor;
+
private ITest defaultReturningProxy;
private ITest exceptionThrowingProxy;
+ private ITest retryingOnceExceptionThrowingProxy;
+
+ private ITest retryingTwiceExceptionThrowingProxy;
+
private static class SignalException extends RuntimeException
{
private static final long serialVersionUID = 1L;
}
+ private static class RetryItException extends RuntimeException
+ {
+ private static final long serialVersionUID = 1L;
+ }
+
enum Status
{
ONE, TWO, THREE, UUUPS, SPECIAL_UUUPS
@@ -69,6 +87,8 @@ public class MonitoringProxyTest
{
void idle(boolean hang);
+ void busyUpdatingActivity();
+
String getString(boolean hang);
boolean getBoolean(boolean hang);
@@ -80,13 +100,24 @@ public class MonitoringProxyTest
Status getSpecialStatus(boolean hang);
void throwSignalException() throws SignalException;
+
+ void worksOnSecondInvocation() throws RetryItException;
+
+ void worksOnThirdInvocation() throws RetryItException;
}
private final static Pattern THREAD_NAME_PATTERN =
- Pattern.compile("Monitoring Proxy-T[0-9]+::main::" + THREAD_NAME);
+ Pattern.compile("Monitoring Proxy-T[0-9]+::main::" + THREAD_NAME);
private class TestImpl implements ITest
{
+ private final IActivityObserver observer;
+
+ TestImpl(IActivityObserver observer)
+ {
+ this.observer = observer;
+ }
+
private void hang(boolean hang)
{
if (hang)
@@ -110,6 +141,18 @@ public class MonitoringProxyTest
hang(hang);
}
+ public void busyUpdatingActivity()
+ {
+ checkThreadName();
+ threadToStop = Thread.currentThread();
+ final long timeToHangMillis = (long) (TIMEOUT_MILLIS * 1.5);
+ final long start = System.currentTimeMillis();
+ while (System.currentTimeMillis() - start < timeToHangMillis)
+ {
+ observer.update();
+ }
+ }
+
public boolean getBoolean(boolean hang)
{
checkThreadName();
@@ -150,20 +193,58 @@ public class MonitoringProxyTest
checkThreadName();
throw new SignalException();
}
+
+ int invocationCount1 = 0;
+
+ public void worksOnSecondInvocation() throws RetryItException
+ {
+ checkThreadName();
+ if (++invocationCount1 < 2)
+ {
+ throw new RetryItException();
+ }
+ }
+
+ int invocationCount2 = 0;
+
+ public void worksOnThirdInvocation() throws RetryItException
+ {
+ checkThreadName();
+ if (++invocationCount2 < 3)
+ {
+ throw new RetryItException();
+ }
+ }
+
}
- @BeforeTest
+ @BeforeClass
public void createMonitoringProxy() throws NoSuchMethodException
{
+ final ISimpleLogger logger = new ConsoleLogger();
+ observerSensor = new RecordingActivityObserverSensor();
defaultReturningProxy =
- MonitoringProxy.create(ITest.class, new TestImpl()).timeoutMillis(TIMEOUT_MILLIS)
- .errorValueOnTimeout().name(THREAD_NAME).errorTypeValueMapping(
- Status.class, Status.UUUPS).errorMethodValueMapping(
+ MonitoringProxy.create(ITest.class, new TestImpl(observerSensor)).timing(
+ TimingParameters.createNoRetries(TIMEOUT_MILLIS)).errorValueOnTimeout()
+ .name(THREAD_NAME).errorTypeValueMapping(Status.class, Status.UUUPS)
+ .errorMethodValueMapping(
ITest.class.getMethod("getSpecialStatus", new Class<?>[]
- { Boolean.TYPE }), Status.SPECIAL_UUUPS).get();
+ { Boolean.TYPE }), Status.SPECIAL_UUUPS).sensor(observerSensor)
+ .errorLog(logger).get();
exceptionThrowingProxy =
- MonitoringProxy.create(ITest.class, new TestImpl()).timeoutMillis(TIMEOUT_MILLIS)
- .name(THREAD_NAME).get();
+ MonitoringProxy.create(ITest.class, new TestImpl(observerSensor)).timing(
+ TimingParameters.createNoRetries(TIMEOUT_MILLIS)).name(THREAD_NAME).sensor(
+ observerSensor).errorLog(logger).get();
+ retryingOnceExceptionThrowingProxy =
+ MonitoringProxy.create(ITest.class, new TestImpl(observerSensor)).timing(
+ TimingParameters.create(TIMEOUT_MILLIS, 1, 0L)).name(THREAD_NAME).sensor(
+ observerSensor).exceptionClassSuitableForRetrying(RetryItException.class)
+ .errorLog(logger).get();
+ retryingTwiceExceptionThrowingProxy =
+ MonitoringProxy.create(ITest.class, new TestImpl(observerSensor)).timing(
+ TimingParameters.create(TIMEOUT_MILLIS, 2, 0L)).name(THREAD_NAME).sensor(
+ observerSensor).exceptionClassSuitableForRetrying(RetryItException.class)
+ .errorLog(logger).get();
}
@SuppressWarnings("deprecation")
@@ -176,7 +257,14 @@ public class MonitoringProxyTest
t.stop();
}
}
-
+
+ @BeforeMethod
+ @AfterClass
+ public void clearThreadInterruptionState()
+ {
+ Thread.interrupted();
+ }
+
@Test
public void testVoid()
{
@@ -207,6 +295,12 @@ public class MonitoringProxyTest
exceptionThrowingProxy.idle(true);
}
+ @Test(groups = "slow")
+ public void testNoTimeoutDueToSensorUpdate()
+ {
+ exceptionThrowingProxy.busyUpdatingActivity();
+ }
+
@Test
public void testGetStringNullReturningPolicy()
{
@@ -283,8 +377,8 @@ public class MonitoringProxyTest
public void testInterruptTheUninterruptableThrowsException()
{
final ITest proxy =
- MonitoringProxy.create(ITest.class, new TestImpl()).timeoutMillis(1000L).name(
- THREAD_NAME).get();
+ MonitoringProxy.create(ITest.class, new TestImpl(observerSensor)).timing(
+ TimingParameters.create(1000L)).name(THREAD_NAME).get();
final Thread currentThread = Thread.currentThread();
final Timer timer = new Timer();
timer.schedule(new TimerTask()
@@ -295,7 +389,7 @@ public class MonitoringProxyTest
currentThread.interrupt();
}
}, 50L);
- // This call would not be interruptable if it wasn't proxied, but we get a StopException
+ // This call would not be interruptible if it wasn't proxied, but we get a StopException
// from the proxy.
proxy.idle(true);
timer.cancel();
@@ -306,9 +400,9 @@ public class MonitoringProxyTest
{
final String defaultReturnValue = "That's the default return value.";
final ITest proxy =
- MonitoringProxy.create(ITest.class, new TestImpl()).timeoutMillis(1000L).name(
- THREAD_NAME).errorValueOnInterrupt().errorTypeValueMapping(String.class,
- defaultReturnValue).get();
+ MonitoringProxy.create(ITest.class, new TestImpl(observerSensor)).timing(
+ TimingParameters.create(1000L)).name(THREAD_NAME).errorValueOnInterrupt()
+ .errorTypeValueMapping(String.class, defaultReturnValue).get();
final Thread currentThread = Thread.currentThread();
final Timer timer = new Timer();
timer.schedule(new TimerTask()
@@ -324,4 +418,28 @@ public class MonitoringProxyTest
assertEquals(defaultReturnValue, proxy.getString(true));
timer.cancel();
}
+
+ @Test(expectedExceptions = RetryItException.class)
+ public void testNoRetryFailOnce()
+ {
+ exceptionThrowingProxy.worksOnSecondInvocation();
+ }
+
+ @Test
+ public void testRetryOnceFailOnce()
+ {
+ retryingOnceExceptionThrowingProxy.worksOnSecondInvocation();
+ }
+
+ @Test(expectedExceptions = RetryItException.class)
+ public void testRetryOnceFailTwice()
+ {
+ retryingOnceExceptionThrowingProxy.worksOnThirdInvocation();
+ }
+
+ @Test(groups = "slow")
+ public void testRetryTwiceFailTwice()
+ {
+ retryingTwiceExceptionThrowingProxy.worksOnThirdInvocation();
+ }
}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/FileOperationsTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/FileOperationsTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..20e0c7460bbda0f2a7c89d0790861925cf62c9ba
--- /dev/null
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/FileOperationsTest.java
@@ -0,0 +1,158 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.TimingParameters;
+import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities;
+import ch.systemsx.cisd.common.concurrent.IActivityObserver;
+import ch.systemsx.cisd.common.concurrent.MonitoringProxy;
+import ch.systemsx.cisd.common.concurrent.RecordingActivityObserverSensor;
+import ch.systemsx.cisd.common.exceptions.TimeoutException;
+import ch.systemsx.cisd.common.exceptions.WrappedIOException;
+import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
+
+/**
+ * Test cases for {@link FileOperations}.
+ *
+ * @author Bernd Rinn
+ */
+public class FileOperationsTest
+{
+
+ static class HangingIInputStream implements IInputStream
+ {
+ private final long hangMillis;
+
+ HangingIInputStream(long hangMillis)
+ {
+ this.hangMillis = hangMillis;
+ }
+
+ private void hang()
+ {
+ ConcurrencyUtilities.sleep(hangMillis);
+ }
+
+ public int available() throws WrappedIOException
+ {
+ hang();
+ return 0;
+ }
+
+ public void close() throws WrappedIOException
+ {
+ hang();
+ }
+
+ public void mark(int readlimit)
+ {
+ hang();
+ }
+
+ public boolean markSupported()
+ {
+ hang();
+ return false;
+ }
+
+ public int read() throws WrappedIOException
+ {
+ hang();
+ return 0;
+ }
+
+ public int read(byte[] b) throws WrappedIOException
+ {
+ hang();
+ return 0;
+ }
+
+ public int read(byte[] b, int off, int len) throws WrappedIOException
+ {
+ hang();
+ return 0;
+ }
+
+ public void reset() throws WrappedIOException
+ {
+ hang();
+ }
+
+ public long skip(long n) throws WrappedIOException
+ {
+ hang();
+ return 0;
+ }
+
+ }
+
+ static class HangingFileOperations extends FileOperations
+ {
+
+ private final long hangMillis;
+
+ HangingFileOperations(TimingParameters parameters, IActivityObserver observer,
+ long hangMillis)
+ {
+ super(parameters, observer);
+ this.hangMillis = hangMillis;
+ }
+
+ @Override
+ IInputStream internalGetIInputStream(File file) throws FileNotFoundException
+ {
+ return new HangingIInputStream(hangMillis);
+ }
+
+ }
+
+ private static IFileOperations create(TimingParameters parameters, long hangMillis)
+ {
+ final RecordingActivityObserverSensor observerSensor =
+ new RecordingActivityObserverSensor();
+ return MonitoringProxy.create(
+ IFileOperations.class,
+ new HangingFileOperations(parameters, observerSensor,
+ hangMillis)).timing(parameters).sensor(
+ observerSensor).errorLog(new Log4jSimpleLogger(FileOperations.operationLog)).name(
+ "remote file operations").get();
+ }
+
+ @Test(groups = "slow", expectedExceptions = TimeoutException.class)
+ public void testNoTimeoutOnInputStream() throws IOException
+ {
+ final IFileOperations ops = create(TimingParameters.createNoRetries(50L), 100L);
+ final InputStream is = ops.getInputStream(new File("bla.txt"));
+ is.read(); // times out
+ }
+
+ @Test(groups = "slow")
+ public void testTimeoutOnInputStream() throws IOException
+ {
+ final IFileOperations ops = create(TimingParameters.createNoRetries(50L), 20L);
+ final InputStream is = ops.getInputStream(new File("bla.txt"));
+ is.read(); // doesn't time out
+ }
+
+}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/QueueingPathRemoverServiceTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/QueueingPathRemoverServiceTest.java
new file mode 100644
index 0000000000000000000000000000000000000000..8941ec0e10c947d0779fbd08d32bf716f26d9f37
--- /dev/null
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/QueueingPathRemoverServiceTest.java
@@ -0,0 +1,156 @@
+/*
+ * Copyright 2008 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.filesystem;
+
+import java.io.File;
+import java.io.FilenameFilter;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.BeforeTest;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities;
+import ch.systemsx.cisd.common.logging.LogInitializer;
+import ch.systemsx.cisd.common.utilities.FileUtilities;
+
+import static org.testng.AssertJUnit.*;
+
+/**
+ * Test cases for the {@link QueueingPathRemoverService}.
+ *
+ * @author Bernd Rinn
+ */
+public class QueueingPathRemoverServiceTest
+{
+
+ private static final long WAIT_MILLIS = 120L;
+
+ private static final long DELAY_MILLIS = 1L;
+
+ private static final File wdRootDirectory =
+ new File("targets" + File.separator + "unit-test-wd");
+
+ private static final File workingDirectory = new File(wdRootDirectory, "ShredderTest");
+
+ private static final File queueFile = new File(workingDirectory, "qfile.dat");
+
+ private static final FilenameFilter SHREDDER_FILTER = new FilenameFilter()
+ {
+ public boolean accept(File dir, String name)
+ {
+ return name.startsWith(QueueingPathRemoverService.SHREDDER_PREFIX);
+ }
+ };
+
+ private final QueueingPathRemoverService shredder = QueueingPathRemoverService.getInstance();
+
+ @BeforeTest
+ public void setUp()
+ {
+ LogInitializer.init();
+ queueFile.deleteOnExit();
+ workingDirectory.mkdirs();
+ workingDirectory.deleteOnExit();
+ QueueingPathRemoverService.start(queueFile);
+ }
+
+ @BeforeMethod
+ public void cleanUp() throws IOException
+ {
+ for (File f : workingDirectory.listFiles(SHREDDER_FILTER))
+ {
+ FileUtilities.deleteRecursively(f);
+ assertFalse(f.exists());
+ }
+ }
+
+ @Test(groups = "slow")
+ public void testShredderFile() throws IOException
+ {
+ final File f = new File(workingDirectory, "someFile");
+ f.createNewFile();
+ f.deleteOnExit();
+ assertTrue(f.exists());
+ shredder.removeRecursively(f);
+ assertFalse(f.exists());
+ ConcurrencyUtilities.sleep(WAIT_MILLIS);
+ assertEquals(0, workingDirectory.list(SHREDDER_FILTER).length);
+ }
+
+ @Test(groups = "slow")
+ public void testShredderDirectory() throws IOException
+ {
+ final File f = new File(workingDirectory, "someDir");
+ f.mkdir();
+ for (int i = 0; i < 100; ++i)
+ {
+ (new File(f, "d" + i)).mkdir();
+ }
+ for (int i = 0; i < 100; ++i)
+ {
+ (new File(f, "f" + i)).createNewFile();
+ }
+ assertTrue(f.exists());
+ shredder.removeRecursively(f);
+ assertFalse(f.exists());
+ ConcurrencyUtilities.sleep(WAIT_MILLIS);
+ assertEquals(0, workingDirectory.list(SHREDDER_FILTER).length);
+ }
+
+ @Test(groups = "slow")
+ public void testManyFiles() throws IOException
+ {
+ final List<File> list = new ArrayList<File>();
+ for (int i = 0; i < 100; ++i)
+ {
+ final File f = new File(workingDirectory, "oneOfMany" + i);
+ f.createNewFile();
+ list.add(f);
+ }
+ for (File f : list)
+ {
+ shredder.removeRecursively(f);
+ assertFalse(f.exists());
+ }
+ ConcurrencyUtilities.sleep(WAIT_MILLIS);
+ assertEquals(0, workingDirectory.list(SHREDDER_FILTER).length);
+ }
+
+ @Test(groups = "slow")
+ public void testManyFilesSlowlyComingIn() throws IOException
+ {
+ final List<File> list = new ArrayList<File>();
+ for (int i = 0; i < 100; ++i)
+ {
+ final File f = new File(workingDirectory, "oneOfMany" + i);
+ f.createNewFile();
+ list.add(f);
+ }
+ for (File f : list)
+ {
+ shredder.removeRecursively(f);
+ ConcurrencyUtilities.sleep(DELAY_MILLIS);
+ assertFalse(f.exists());
+ }
+ ConcurrencyUtilities.sleep(WAIT_MILLIS);
+ assertEquals(0, workingDirectory.list(SHREDDER_FILTER).length);
+ }
+
+}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
index 5ce6c9f5a077c18b11d42bb1d3177cc761a4775d..e8eb15f7271626fb7498dd97bd6f1fa3ded114aa 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/filesystem/rsync/RsyncCopierTest.java
@@ -25,8 +25,11 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
+import java.util.Timer;
+import java.util.TimerTask;
import org.apache.commons.io.FileUtils;
+import org.apache.log4j.Logger;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -35,8 +38,9 @@ import ch.rinn.restrictions.Friend;
import ch.systemsx.cisd.common.collections.CollectionIO;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.exceptions.StatusFlag;
-import ch.systemsx.cisd.common.filesystem.rsync.RsyncCopier;
-import ch.systemsx.cisd.common.filesystem.rsync.RsyncExitValueTranslator;
+import ch.systemsx.cisd.common.exceptions.StopException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.logging.LogInitializer;
import ch.systemsx.cisd.common.test.StoringUncaughtExceptionHandler;
import ch.systemsx.cisd.common.utilities.FileUtilities;
@@ -49,6 +53,10 @@ import ch.systemsx.cisd.common.utilities.FileUtilities;
@Friend(toClasses = RsyncCopier.class)
public final class RsyncCopierTest
{
+ private static final Logger operationLog =
+ LogFactory.getLogger(LogCategory.OPERATION, RsyncCopierTest.class);
+
+ private static final long SLEEP_MILLIS = 1000L;
private static final File unitTestRootDirectory =
new File("targets" + File.separator + "unit-test-wd");
@@ -64,6 +72,28 @@ public final class RsyncCopierTest
private final StoringUncaughtExceptionHandler exceptionHandler =
new StoringUncaughtExceptionHandler();
+ private File createExecutable(String name, String... lines) throws IOException,
+ InterruptedException
+ {
+ final File executable = new File(workingDirectory, name);
+ executable.delete();
+ CollectionIO.writeIterable(executable, Arrays.asList(lines));
+ Runtime.getRuntime().exec(String.format("/bin/chmod +x %s", executable.getPath()))
+ .waitFor();
+ executable.deleteOnExit();
+ return executable;
+ }
+
+ private final String sleepyMessage = "I am feeling sooo sleepy...";
+
+ private File createSleepingRsyncExecutable(String name, long millisToSleep) throws IOException,
+ InterruptedException
+ {
+ return createExecutable(name, "#! /bin/sh",
+ "if [ \"$1\" = \"--version\" ]; then echo \"rsync version 3.0.3\"; exit 0; fi",
+ "echo " + sleepyMessage, "sleep " + (millisToSleep / 1000.0f), "exit 0");
+ }
+
@BeforeClass
public void init()
{
@@ -410,4 +440,39 @@ public final class RsyncCopierTest
return createRsync("2.6.9", "/bin/sleep " + seconds);
}
+ @Test(groups =
+ { "slow", "requires_unix" }, expectedExceptions = StopException.class)
+ public void testStopRsyncCopierCopyImmutably() throws Exception
+ {
+ final File rsyncExecutable = createSleepingRsyncExecutable("rsync", SLEEP_MILLIS);
+ assertTrue(rsyncExecutable.exists());
+ final File source = new File(unitTestRootDirectory, "a");
+ source.mkdir();
+ source.deleteOnExit();
+ assertTrue(source.isDirectory());
+ final File destination = new File(unitTestRootDirectory, "b");
+ destination.mkdir();
+ destination.deleteOnExit();
+ assertTrue(destination.isDirectory());
+ final RsyncCopier copier = new RsyncCopier(rsyncExecutable);
+ final Thread thisThread = Thread.currentThread();
+ final Timer timer = new Timer();
+ try
+ {
+ timer.schedule(new TimerTask()
+ {
+ @Override
+ public void run()
+ {
+ operationLog.info("Interrupting thread");
+ thisThread.interrupt();
+ }
+ }, SLEEP_MILLIS / 10);
+ copier.copyDirectoryImmutably(source, destination, null);
+ } finally
+ {
+ timer.cancel();
+ }
+ }
+
}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/process/ProcessExecutionHelperTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/process/ProcessExecutionHelperTest.java
index 9747f1f17ba4754fac229d639daeca333642597a..bfa1a656c7450afd876a76f5c71ab83881a0071a 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/process/ProcessExecutionHelperTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/process/ProcessExecutionHelperTest.java
@@ -28,6 +28,7 @@ import java.util.TimerTask;
import java.util.concurrent.atomic.AtomicReference;
import org.apache.log4j.Logger;
+import org.testng.annotations.AfterClass;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
import org.testng.annotations.Test;
@@ -69,6 +70,7 @@ public class ProcessExecutionHelperTest
final File executable = new File(workingDirectory, name);
executable.delete();
CollectionIO.writeIterable(executable, Arrays.asList(lines));
+ Thread.interrupted(); // Paranoia: clear interrupted state.
Runtime.getRuntime().exec(String.format("/bin/chmod +x %s", executable.getPath()))
.waitFor();
executable.deleteOnExit();
@@ -106,6 +108,13 @@ public class ProcessExecutionHelperTest
workingDirectory.deleteOnExit();
}
+ @BeforeMethod
+ @AfterClass
+ public void clearThreadInterruptionState()
+ {
+ Thread.interrupted();
+ }
+
@Test(groups =
{ "requires_unix" })
public void testExecutionOKWithoutTimeOut() throws Exception
@@ -191,6 +200,37 @@ public class ProcessExecutionHelperTest
}
}
+ @Test(groups =
+ { "requires_unix", "slow" }, expectedExceptions =
+ { StopException.class })
+ public void testSleepyExecutionGetsStoppedUsingRunUnblocking() throws Exception
+ {
+ final Thread thisThread = Thread.currentThread();
+ final Timer timer = new Timer();
+ try
+ {
+ timer.schedule(new TimerTask()
+ {
+ @Override
+ public void run()
+ {
+ thisThread.interrupt();
+ }
+ }, WATCHDOG_WAIT_MILLIS / 10);
+ final File dummyExec =
+ createSleepingExecutable("dummySleepyFailedWithTimeOut.sh",
+ 2 * WATCHDOG_WAIT_MILLIS);
+ final IProcessHandler processHandler =
+ ProcessExecutionHelper.runUnblocking(
+ Arrays.asList(dummyExec.getAbsolutePath()),
+ OutputReadingStrategy.ON_ERROR, operationLog, machineLog);
+ processHandler.getResult();
+ } finally
+ {
+ timer.cancel();
+ }
+ }
+
@Test(groups =
{ "requires_unix", "slow" })
public void testSleepyExecutionGetsInterrupted() throws Exception
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/AbstractFileSystemTestCase.java b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/AbstractFileSystemTestCase.java
index 739f9fdedfca16587f68e4c1e0fa6420c7a3d5f5..0d33bfc6f8b1aa0167424469a5117b55ce333778 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/AbstractFileSystemTestCase.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/AbstractFileSystemTestCase.java
@@ -77,6 +77,6 @@ public abstract class AbstractFileSystemTestCase
{
return;
}
- FileUtils.deleteDirectory(workingDirectory);
+ FileUtilities.deleteRecursively(workingDirectory);
}
}
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTaskTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTaskTest.java
index 358749b0cbfd775126061e70177bba82f1dd43b5..8640b887e0f245ca685d7a37b261d5c2ab082981 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTaskTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/DirectoryScanningTimerTaskTest.java
@@ -17,7 +17,7 @@
package ch.systemsx.cisd.common.utilities;
import static ch.systemsx.cisd.common.utilities.FileUtilities.ACCEPT_ALL_FILTER;
-import static org.testng.AssertJUnit.assertEquals;
+import static org.testng.AssertJUnit.*;
import java.io.File;
import java.io.FileFilter;
@@ -25,6 +25,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
+import java.util.concurrent.atomic.AtomicInteger;
import org.testng.annotations.BeforeClass;
import org.testng.annotations.BeforeMethod;
@@ -93,6 +94,11 @@ public class DirectoryScanningTimerTaskTest
handledPaths.add(path);
path.delete();
}
+
+ public boolean isStopped()
+ {
+ return false;
+ }
}
@BeforeClass
@@ -204,7 +210,7 @@ public class DirectoryScanningTimerTaskTest
assertEquals(1, faulty.size());
assertEquals(someFile.getPath(), faulty.get(0));
// See whether fault_paths resetting works.
- assert faultyPaths.delete();
+ assertTrue(faultyPaths.delete());
myPathHandler.clear(); // Isn't necessary, just for expressing intention.
// See whether faulty_paths settings works.
scanner.run();
@@ -212,6 +218,62 @@ public class DirectoryScanningTimerTaskTest
assertEquals(someFile, myPathHandler.handledPaths.get(0));
}
+ @Test
+ public void testStopped() throws IOException
+ {
+ final DirectoryScanningTimerTask scanner =
+ new DirectoryScanningTimerTask(workingDirectory, ACCEPT_ALL_FILTER, new IPathHandler()
+ {
+ public void handle(File path)
+ {
+ throw new AssertionError("Shouldn't have been called");
+ }
+
+ public boolean isStopped()
+ {
+ return true;
+ }
+ });
+ final File someFile = new File(workingDirectory, "some_file");
+ someFile.createNewFile();
+ assertTrue(someFile.exists());
+ someFile.deleteOnExit();
+ scanner.run();
+ }
+
+ @Test
+ public void testStopInFirstFile() throws IOException
+ {
+ final File faultyPaths =
+ new File(workingDirectory, Constants.FAULTY_PATH_FILENAME);
+ faultyPaths.createNewFile();
+ assertTrue(faultyPaths.exists());
+ final AtomicInteger counter = new AtomicInteger(0);
+ final DirectoryScanningTimerTask scanner =
+ new DirectoryScanningTimerTask(workingDirectory, ACCEPT_ALL_FILTER, new IPathHandler()
+ {
+ boolean stop = false;
+ public void handle(File path)
+ {
+ counter.incrementAndGet();
+ stop = true;
+ }
+
+ public boolean isStopped()
+ {
+ return stop;
+ }
+ });
+ final File someFile = new File(workingDirectory, "some_file");
+ someFile.createNewFile();
+ assertTrue(someFile.exists());
+ final File someOtherFile = new File(workingDirectory, "some_other_file");
+ someOtherFile.deleteOnExit();
+ scanner.run();
+ assertEquals(1, counter.get());
+ assertEquals(0L, faultyPaths.length());
+ }
+
@Test
public void testPathOrder() throws IOException
{
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/FileUtilitiesDeleteRecursivelyTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/FileUtilitiesDeleteRecursivelyTest.java
index 31b7dc9beada3dae953acda331c49745d6e0c47f..ba04e0b6c0e320001736d2fb266b1220941d879a 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/FileUtilitiesDeleteRecursivelyTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/FileUtilitiesDeleteRecursivelyTest.java
@@ -25,8 +25,8 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.testng.annotations.Test;
+import ch.systemsx.cisd.common.concurrent.IActivityObserver;
import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
-import ch.systemsx.cisd.common.utilities.FileUtilities.SimpleActivityObserver;
/**
* Test cases for the <code>deleteRecursively</code> methods of {@link FileUtilities}.
@@ -174,7 +174,7 @@ public final class FileUtilitiesDeleteRecursivelyTest extends AbstractFileSystem
{
return pathname.getAbsolutePath().equals(workingDirectory.getAbsolutePath()) == false;
}
- }, null, new SimpleActivityObserver()
+ }, null, new IActivityObserver()
{
public void update()
{
diff --git a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/FileUtilitiesTest.java b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/FileUtilitiesTest.java
index f83938daf24be44db5530700866db9852b5e4662..943658e52ec2dd4aff74f4bc58d23ef25f4c944a 100644
--- a/common/sourceTest/java/ch/systemsx/cisd/common/utilities/FileUtilitiesTest.java
+++ b/common/sourceTest/java/ch/systemsx/cisd/common/utilities/FileUtilitiesTest.java
@@ -29,6 +29,7 @@ import java.io.FileWriter;
import java.io.IOException;
import java.io.InputStream;
import java.util.Arrays;
+import java.util.HashSet;
import java.util.List;
import java.util.regex.Pattern;
@@ -37,6 +38,7 @@ import org.apache.commons.io.IOUtils;
import org.testng.annotations.Test;
import ch.systemsx.cisd.common.Constants;
+import ch.systemsx.cisd.common.concurrent.IActivityObserver;
import ch.systemsx.cisd.common.exceptions.CheckedExceptionTunnel;
import ch.systemsx.cisd.common.parser.filter.ExcludeEmptyAndCommentLineFilter;
@@ -432,6 +434,72 @@ public final class FileUtilitiesTest extends AbstractFileSystemTestCase
}
+ private class CountingActivityObserver implements IActivityObserver
+ {
+ int count = 0;
+
+ public void update()
+ {
+ ++count;
+ }
+ }
+
+ @Test
+ public void testListFiles() throws IOException
+ {
+ final File dir = new File(workingDirectory, "listFiles");
+ dir.mkdir();
+ final File nonExistentDir = new File(dir, "nonExistent");
+ assertTrue(FileUtilities.listFiles(dir, null, true, null).isEmpty());
+ assertTrue(FileUtilities.listFiles(nonExistentDir, null, true, null).isEmpty());
+ final File subDir = new File(dir, "subdir");
+ subDir.mkdir();
+ assertTrue(FileUtilities.listFiles(dir, null, true, null).isEmpty());
+ assertEquals("subdir", FileUtilities.listDirectories(dir, true, null).get(0).getName());
+ final File f1 = new File(dir, "f1.dat");
+ f1.createNewFile();
+ final File f2 = new File(dir, "f2.bla");
+ f2.createNewFile();
+ final File f3 = new File(subDir, "f3");
+ f3.createNewFile();
+ final File f4 = new File(subDir, "f4.dat");
+ f4.createNewFile();
+ final CountingActivityObserver observer = new CountingActivityObserver();
+
+ final List<File> list1 = FileUtilities.listFiles(dir, null, true, null);
+ assertEquals(4, list1.size());
+ assertEquals(new HashSet<File>(Arrays.asList(f1, f2, f3, f4)), new HashSet<File>(list1));
+
+ final List<File> list2 = FileUtilities.listFiles(dir, null, true, observer);
+ assertEquals(list1, list2);
+ assertTrue("" + observer.count, observer.count >= list2.size());
+
+ observer.count = 0;
+ final List<File> list3 = FileUtilities.listFiles(dir, new String[]
+ { "dat" }, true, observer);
+ assertEquals(2, list3.size());
+ assertEquals(new HashSet<File>(Arrays.asList(f1, f4)), new HashSet<File>(list3));
+ assertTrue("" + observer.count, observer.count >= list3.size());
+
+ final File subDir2 = new File(subDir, "subDir2");
+ subDir2.mkdir();
+ final File subDir3 = new File(dir, "subDir3");
+ subDir3.mkdir();
+ observer.count = 0;
+ final List<File> list4 = FileUtilities.listDirectories(dir, true, observer);
+ assertEquals(3, list4.size());
+ assertEquals(new HashSet<File>(Arrays.asList(subDir, subDir2, subDir3)), new HashSet<File>(
+ list4));
+ assertTrue("" + observer.count, observer.count >= list4.size());
+
+ observer.count = 0;
+ final List<File> list5 = FileUtilities.listDirectories(dir, false, observer);
+ assertEquals(2, list5.size());
+ assertEquals(new HashSet<File>(Arrays.asList(subDir, subDir3)), new HashSet<File>(
+ list5));
+ assertTrue("" + observer.count, observer.count >= list5.size());
+ }
+
@Test
public final void testNormalizeFile() throws IOException
{
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
index 1a0298af053294416863c26eac88ae5a6e9a52fe..3a23f00534696b8d28f1841925edabef5bb5b854 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java
@@ -252,7 +252,7 @@ public final class DataMover
createRemotePathMover(readyToMoveStore, outgoingStore);
final HighwaterMarkDirectoryScanningHandler directoryScanningHandler =
new HighwaterMarkDirectoryScanningHandler(new FaultyPathDirectoryScanningHandler(
- sourceDirectory), outgoingStore.getHighwaterMarkWatcher());
+ sourceDirectory, remoteStoreMover), outgoingStore.getHighwaterMarkWatcher());
final DirectoryScanningTimerTask outgoingMovingTask =
new DirectoryScanningTimerTask(sourceDirectory, FileUtilities.ACCEPT_ALL_FILTER,
remoteStoreMover, directoryScanningHandler);
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java b/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
index 1662daf3f85d5287ce80f06e47986588ef1a3aa3..5b4df75ad7f498b498edb4749080031685584b90 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/IncomingProcessor.java
@@ -159,10 +159,10 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
final File copyInProgressDir = bufferDirs.getCopyInProgressDir();
final HighwaterMarkWatcher highwaterMarkWatcher =
new HighwaterMarkWatcher(bufferDirs.getBufferDirHighwaterMark());
+ final IStoreHandler pathHandler = createIncomingMovingPathHandler();
final HighwaterMarkDirectoryScanningHandler directoryScanningHandler =
new HighwaterMarkDirectoryScanningHandler(new FaultyPathDirectoryScanningHandler(
- copyInProgressDir), highwaterMarkWatcher, copyInProgressDir);
- final IStoreHandler pathHandler = createIncomingMovingPathHandler();
+ copyInProgressDir, pathHandler), highwaterMarkWatcher, copyInProgressDir);
final DirectoryScanningTimerTask movingTask =
new DirectoryScanningTimerTask(
new FileScannedStore(incomingStore, storeItemFilter),
@@ -206,6 +206,12 @@ public class IncomingProcessor implements IRecoverableTimerTaskFactory
moveFromLocalIncoming(extendedFileStore, sourceItem);
}
}
+
+ public boolean isStopped()
+ {
+ return (incomingStore.tryAsExtended() == null) ? remotePathMover.isStopped()
+ : false;
+ }
};
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java b/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
index e7fe31f8391eaee68468f1bcbe58f3c6ef0699c7..5608239b1d8eb0d72d9436118a57fb691e15a0d4 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/LocalProcessor.java
@@ -26,6 +26,7 @@ import org.apache.commons.io.filefilter.RegexFileFilter;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+import ch.systemsx.cisd.common.exceptions.StopException;
import ch.systemsx.cisd.common.logging.ISimpleLogger;
import ch.systemsx.cisd.common.logging.Log4jSimpleLogger;
import ch.systemsx.cisd.common.logging.LogCategory;
@@ -80,6 +81,8 @@ public final class LocalProcessor implements IPathHandler, IRecoverableTimerTask
private final File manualInterventionDir;
+ private boolean stopped = false;
+
LocalProcessor(final Parameters parameters, final LocalBufferDirs bufferDirs,
final IImmutableCopier copier, final IPathMover mover)
{
@@ -249,63 +252,75 @@ public final class LocalProcessor implements IPathHandler, IRecoverableTimerTask
public final void handle(final File path)
{
- final boolean continueProcessing = doMoveManualOrClean(path);
- if (continueProcessing == false)
- {
- // Stop processing
- return;
- }
- File extraTmpCopy = null;
- if (extraCopyDirOrNull != null)
+ stopped = false;
+ try
{
- extraTmpCopy = new File(tempDir, path.getName());
- if (extraTmpCopy.exists())
+ final boolean continueProcessing = doMoveManualOrClean(path);
+ if (continueProcessing == false)
{
- operationLog.warn(String.format(
- "Half-finished extra copy directory '%s' exists - removing it.",
- extraTmpCopy.getAbsolutePath()));
- if (FileUtilities.deleteRecursively(extraTmpCopy) == false)
+ // Stop processing
+ return;
+ }
+ File extraTmpCopy = null;
+ if (extraCopyDirOrNull != null)
+ {
+ extraTmpCopy = new File(tempDir, path.getName());
+ if (extraTmpCopy.exists())
{
- notificationLog.error(String.format(
- "Removal of half-finished extra copy directory '%s' failed.",
+ operationLog.warn(String.format(
+ "Half-finished extra copy directory '%s' exists - removing it.",
extraTmpCopy.getAbsolutePath()));
+ if (FileUtilities.deleteRecursively(extraTmpCopy) == false)
+ {
+ notificationLog.error(String.format(
+ "Removal of half-finished extra copy directory '%s' failed.",
+ extraTmpCopy.getAbsolutePath()));
+ return;
+ }
+ }
+ if (operationLog.isInfoEnabled())
+ {
+ operationLog.info(String.format("Creating extra copy of directory '%s' to '%s'.",
+ path.getAbsolutePath(), tempDir.getAbsoluteFile()));
+ }
+ final boolean ok = copier.copyImmutably(path, tempDir, null);
+ if (ok == false)
+ {
+ notificationLog.error(String.format("Creating extra copy of '%s' failed.", path
+ .getAbsolutePath()));
return;
}
}
- if (operationLog.isInfoEnabled())
+
+ final File movedFile = mover.tryMove(path, outputDir);
+ if (movedFile == null)
{
- operationLog.info(String.format("Creating extra copy of directory '%s' to '%s'.",
- path.getAbsolutePath(), tempDir.getAbsoluteFile()));
- }
- final boolean ok = copier.copyImmutably(path, tempDir, null);
- if (ok == false)
- {
- notificationLog.error(String.format("Creating extra copy of '%s' failed.", path
- .getAbsolutePath()));
+ notificationLog.error(String.format(
+ "Moving '%s' to '%s' for final moving process failed.", path, outputDir));
return;
}
- }
-
- final File movedFile = mover.tryMove(path, outputDir);
- if (movedFile == null)
- {
- notificationLog.error(String.format(
- "Moving '%s' to '%s' for final moving process failed.", path, outputDir));
- return;
- }
-
- if (extraTmpCopy != null)
- {
- assert extraCopyDirOrNull != null;
- final File extraCopy = mover.tryMove(extraTmpCopy, extraCopyDirOrNull);
- if (extraCopy == null)
+
+ if (extraTmpCopy != null)
{
- notificationLog.error(String.format(
- "Moving temporary extra copy '%s' to destination '%s' failed.",
- extraTmpCopy, extraCopyDirOrNull));
+ assert extraCopyDirOrNull != null;
+ final File extraCopy = mover.tryMove(extraTmpCopy, extraCopyDirOrNull);
+ if (extraCopy == null)
+ {
+ notificationLog.error(String.format(
+ "Moving temporary extra copy '%s' to destination '%s' failed.",
+ extraTmpCopy, extraCopyDirOrNull));
+ }
}
+ } catch (StopException ex)
+ {
+ stopped = true;
}
}
+
+ public boolean isStopped()
+ {
+ return stopped;
+ }
//
// IRecoverableTimerTaskFactory
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java
index 44b9f85d62ccf6d2785ea6fde9db08f09ec08214..5f53a886a8651477760dc25620404440a5978a20 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/FileSysOperationsFactory.java
@@ -81,7 +81,7 @@ public class FileSysOperationsFactory implements IFileSysOperationsFactory
public final IImmutableCopier getImmutableCopier()
{
final File rsyncExecutable = findRsyncExecutable();
- return FastRecursiveHardLinkMaker.tryCreate(rsyncExecutable, true);
+ return FastRecursiveHardLinkMaker.create(rsyncExecutable);
}
public final IPathCopier getCopier(final boolean requiresDeletionBeforeCreation)
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathRemover.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathRemover.java
index c99f2fc67e04d7f26a2dc2a540160fb194a8c4d7..b6a8c7b4775f07d2560669158a023a0fdf0f54bb 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathRemover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/RetryingPathRemover.java
@@ -28,15 +28,15 @@ import ch.rinn.restrictions.Private;
import ch.systemsx.cisd.common.Constants;
import ch.systemsx.cisd.common.concurrent.ConcurrencyUtilities;
import ch.systemsx.cisd.common.concurrent.ExecutionResult;
+import ch.systemsx.cisd.common.concurrent.IActivityObserver;
import ch.systemsx.cisd.common.concurrent.InactivityMonitor;
import ch.systemsx.cisd.common.concurrent.NamingThreadPoolExecutor;
-import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IActivitySensor;
+import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IDescribingActivitySensor;
import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IInactivityObserver;
import ch.systemsx.cisd.common.exceptions.Status;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
import ch.systemsx.cisd.common.utilities.FileUtilities;
-import ch.systemsx.cisd.common.utilities.FileUtilities.SimpleActivityObserver;
import ch.systemsx.cisd.datamover.filesystem.intf.IPathRemover;
/**
@@ -125,9 +125,9 @@ final class RetryingPathRemover implements IPathRemover
}
@Private
- static class DeleteActivityDetector implements IActivitySensor, SimpleActivityObserver
+ static class DeleteActivityDetector implements IDescribingActivitySensor, IActivityObserver
{
- private volatile long lastActivityTime = System.currentTimeMillis();
+ private volatile long lastActivityMillis = System.currentTimeMillis();
private final File path;
@@ -139,18 +139,23 @@ final class RetryingPathRemover implements IPathRemover
// called each time when one file gets deleted
synchronized public void update()
{
- lastActivityTime = System.currentTimeMillis();
+ lastActivityMillis = System.currentTimeMillis();
}
synchronized public String describeInactivity(long now)
{
return "No delete activity of path " + path.getPath() + " for "
- + DurationFormatUtils.formatDurationHMS(now - lastActivityTime);
+ + DurationFormatUtils.formatDurationHMS(now - lastActivityMillis);
}
- synchronized public long getTimeOfLastActivityMoreRecentThan(long thresholdMillis)
+ synchronized public long getLastActivityMillisMoreRecentThan(long thresholdMillis)
{
- return lastActivityTime;
+ return lastActivityMillis;
+ }
+
+ synchronized public boolean hasActivityMoreRecentThan(long thresholdMillis)
+ {
+ return (System.currentTimeMillis() - lastActivityMillis) < thresholdMillis;
}
}
@@ -170,7 +175,7 @@ final class RetryingPathRemover implements IPathRemover
}
@Private
- Boolean executeAndMonitor(final IActivitySensor sensor, final Callable<Boolean> deleteCallable,
+ Boolean executeAndMonitor(final IDescribingActivitySensor sensor, final Callable<Boolean> deleteCallable,
final long inactivityThresholdMillis)
{
final Future<Boolean> deleteFuture = executor.submit(deleteCallable);
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
index 2f163b73cc5e34948542aa63a3ba4161dc9c3ac3..e177aeffd6a7eb4802334ed1712cc7cc4d7ee484 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemotePathMover.java
@@ -62,6 +62,9 @@ public final class RemotePathMover implements IStoreHandler
private static final String MOVING_PATH_TO_REMOTE_FAILED_TEMPLATE =
"Moving path '%s' to remote directory '%s' failed.";
+ private static final String MOVING_PATH_TO_REMOTE_STOPPED_TEMPLATE =
+ "Moving path '%s' to remote directory '%s' was stopped.";
+
private static final String REMOVING_LOCAL_PATH_FAILED_TEMPLATE =
"Removing local path '%s' failed (%s).";
@@ -97,6 +100,8 @@ public final class RemotePathMover implements IStoreHandler
private final int maximalNumberOfRetries;
+ private boolean stopped;
+
/**
* Creates a <var>PathRemoteMover</var>.
*
@@ -108,8 +113,8 @@ public final class RemotePathMover implements IStoreHandler
* @throws ConfigurationFailureException If the destination directory is not fully accessible.
*/
public RemotePathMover(final IFileStore sourceDirectory, final IFileStore destinationDirectory,
- final IStoreCopier copier,
- final ITimingParameters timingParameters) throws ConfigurationFailureException
+ final IStoreCopier copier, final ITimingParameters timingParameters)
+ throws ConfigurationFailureException
{
assert sourceDirectory != null;
assert destinationDirectory != null;
@@ -123,6 +128,7 @@ public final class RemotePathMover implements IStoreHandler
this.intervallToWaitAfterFailure = timingParameters.getIntervalToWaitAfterFailure();
this.maximalNumberOfRetries = timingParameters.getMaximalNumberOfRetries();
this.inactivityPeriodMillis = timingParameters.getInactivityPeriodMillis();
+ this.stopped = false;
assert intervallToWaitAfterFailure >= 0;
assert maximalNumberOfRetries >= 0;
@@ -143,8 +149,9 @@ public final class RemotePathMover implements IStoreHandler
public void update(long inactiveSinceMillis,
String descriptionOfInactivity)
{
- operationLog.warn(String.format(TERMINATING_COPIER_LOG_TEMPLATE,
- copier.getClass().getName(), descriptionOfInactivity));
+ operationLog.warn(String.format(
+ TERMINATING_COPIER_LOG_TEMPLATE, copier.getClass()
+ .getName(), descriptionOfInactivity));
copier.terminate();
}
}, inactivityPeriodMillis, true);
@@ -336,6 +343,11 @@ public final class RemotePathMover implements IStoreHandler
int tryCount = 0;
do
{
+ if (Thread.interrupted())
+ {
+ stopped = true;
+ break;
+ }
if (operationLog.isInfoEnabled())
{
if (tryCount > 0) // This is a retry
@@ -384,12 +396,24 @@ public final class RemotePathMover implements IStoreHandler
Thread.sleep(intervallToWaitAfterFailure);
} catch (final InterruptedException e)
{
- // We don't expect to get interrupted, but even if, there is no need to handle this
- // here.
+ stopped = true;
+ break;
}
} while (true);
- notificationLog.error(String.format(MOVING_PATH_TO_REMOTE_FAILED_TEMPLATE,
- getSrcPath(item), destinationDirectory));
+ if (stopped)
+ {
+ operationLog.warn(String.format(MOVING_PATH_TO_REMOTE_STOPPED_TEMPLATE,
+ getSrcPath(item), destinationDirectory));
+ } else
+ {
+ notificationLog.error(String.format(MOVING_PATH_TO_REMOTE_FAILED_TEMPLATE,
+ getSrcPath(item), destinationDirectory));
+ }
+ }
+
+ public boolean isStopped()
+ {
+ return stopped;
}
}
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemoteStoreCopyActivitySensor.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemoteStoreCopyActivitySensor.java
index f434f8599d3011df18151f702983710863e92716..37addf69193b59b9f6e618ae341ef0c8cf262aa5 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemoteStoreCopyActivitySensor.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/remote/RemoteStoreCopyActivitySensor.java
@@ -18,7 +18,7 @@ package ch.systemsx.cisd.datamover.filesystem.remote;
import org.apache.log4j.Logger;
-import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IActivitySensor;
+import ch.systemsx.cisd.common.concurrent.InactivityMonitor.IDescribingActivitySensor;
import ch.systemsx.cisd.common.exceptions.StatusWithResult;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -27,15 +27,15 @@ import ch.systemsx.cisd.common.utilities.StoreItem;
import ch.systemsx.cisd.datamover.filesystem.intf.IFileStore;
/**
- * A {@link IActivitySensor} that senses changes in copy operations to a {@link StoreItem} in a
- * remote store.
+ * A {@link IDescribingActivitySensor} that senses changes in copy operations to a {@link StoreItem}
+ * in a remote store.
*
* @author Bernd Rinn
*/
public class RemoteStoreCopyActivitySensor extends AbstractCopyActivitySensor
{
private final static Logger machineLog =
- LogFactory.getLogger(LogCategory.MACHINE, RemoteStoreCopyActivitySensor.class);
+ LogFactory.getLogger(LogCategory.MACHINE, RemoteStoreCopyActivitySensor.class);
private final IFileStore destinationStore;
diff --git a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
index b3d8c25614d0cde297f6fbe89c38b68830e7e24b..91ef614321ccb927ecbf96a9dd059f39587ef60c 100644
--- a/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
+++ b/datamover/source/java/ch/systemsx/cisd/datamover/filesystem/store/FileStoreRemoteMounted.java
@@ -16,6 +16,7 @@
package ch.systemsx.cisd.datamover.filesystem.store;
+import ch.systemsx.cisd.common.TimingParameters;
import ch.systemsx.cisd.common.concurrent.MonitoringProxy;
import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
@@ -58,8 +59,8 @@ public final class FileStoreRemoteMounted extends AbstractFileStore
super(file, description, factory);
this.localImpl = new FileStoreLocal(file, description, factory);
this.localImplMonitored =
- MonitoringProxy.create(IFileStore.class, localImpl).timeoutMillis(
- lastChangedTimeoutMillis).get();
+ MonitoringProxy.create(IFileStore.class, localImpl).timing(
+ TimingParameters.create(lastChangedTimeoutMillis)).get();
}
//
diff --git a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/RemoteStoreCopyActivitySensorTest.java b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/RemoteStoreCopyActivitySensorTest.java
index ee847deb628b3cbb62ab31160ef41f5963d03d9b..f1347e7a627821c2f2c8b9bf4903f1fc27f8005b 100644
--- a/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/RemoteStoreCopyActivitySensorTest.java
+++ b/datamover/sourceTest/java/ch/systemsx/cisd/datamover/filesystem/remote/RemoteStoreCopyActivitySensorTest.java
@@ -92,7 +92,7 @@ public class RemoteStoreCopyActivitySensorTest
});
final long delta =
System.currentTimeMillis()
- - sensorUnderTest.getTimeOfLastActivityMoreRecentThan(THRESHOLD);
+ - sensorUnderTest.getLastActivityMillisMoreRecentThan(THRESHOLD);
assertTrue("Delta is " + delta, delta < MAX_DELTA);
final String msg = sensorUnderTest.describeInactivity(System.currentTimeMillis());
assertTrue(
@@ -113,7 +113,7 @@ public class RemoteStoreCopyActivitySensorTest
}
});
final long now = System.currentTimeMillis();
- final long delta = now - sensorUnderTest.getTimeOfLastActivityMoreRecentThan(THRESHOLD);
+ final long delta = now - sensorUnderTest.getLastActivityMillisMoreRecentThan(THRESHOLD);
assertTrue("Delta is " + delta, delta < MAX_DELTA);
assertEquals("Error: Unable to determine the time of write activity on "
+ "item 'I am probed' in store 'iFileStore'\nERROR message", sensorUnderTest
@@ -133,14 +133,14 @@ public class RemoteStoreCopyActivitySensorTest
});
ConcurrencyUtilities.sleep(10L);
final long now = System.currentTimeMillis();
- final long lastActivity1 = sensorUnderTest.getTimeOfLastActivityMoreRecentThan(THRESHOLD);
+ final long lastActivity1 = sensorUnderTest.getLastActivityMillisMoreRecentThan(THRESHOLD);
final long delta = lastActivity1 - now;
assertTrue("Delta is " + delta, delta < MAX_DELTA);
ConcurrencyUtilities.sleep(10L);
- final long lastActivity2 = sensorUnderTest.getTimeOfLastActivityMoreRecentThan(THRESHOLD);
+ final long lastActivity2 = sensorUnderTest.getLastActivityMillisMoreRecentThan(THRESHOLD);
assertEquals(lastActivity1, lastActivity2);
ConcurrencyUtilities.sleep(10L);
- final long lastActivity3 = sensorUnderTest.getTimeOfLastActivityMoreRecentThan(THRESHOLD);
+ final long lastActivity3 = sensorUnderTest.getLastActivityMillisMoreRecentThan(THRESHOLD);
assertEquals(lastActivity1, lastActivity3);
}
@@ -159,14 +159,14 @@ public class RemoteStoreCopyActivitySensorTest
});
ConcurrencyUtilities.sleep(10L);
final long now1 = System.currentTimeMillis();
- final long lastActivity1 = sensorUnderTest.getTimeOfLastActivityMoreRecentThan(THRESHOLD);
+ final long lastActivity1 = sensorUnderTest.getLastActivityMillisMoreRecentThan(THRESHOLD);
assertTrue("Delta=" + (lastActivity1 - now1), lastActivity1 - now1 < MAX_DELTA);
ConcurrencyUtilities.sleep(10L);
final long now2 = System.currentTimeMillis();
- final long lastActivity2 = sensorUnderTest.getTimeOfLastActivityMoreRecentThan(THRESHOLD);
+ final long lastActivity2 = sensorUnderTest.getLastActivityMillisMoreRecentThan(THRESHOLD);
assertTrue("Delta=" + (lastActivity2 - now2), lastActivity2 - now2 < MAX_DELTA);
ConcurrencyUtilities.sleep(10L);
- final long lastActivity3 = sensorUnderTest.getTimeOfLastActivityMoreRecentThan(THRESHOLD);
+ final long lastActivity3 = sensorUnderTest.getLastActivityMillisMoreRecentThan(THRESHOLD);
assertEquals(lastActivity1, lastActivity3);
}