change: - Make the shutdown hook in DataMover more robust.

SVN: 6920

datamover/source/java/ch/systemsx/cisd/datamover/DataMover.java

@@ -65,10 +65,11 @@ public final class DataMover
     @Private
     static final String RECOVERY_MARKER_FIILENAME = Constants.MARKER_PREFIX + "recovery";
 
-    @Private static final String PROCESS_MARKER_PREFIX = Constants.MARKER_PREFIX + "thread_";
-    
+    @Private
+    static final String PROCESS_MARKER_PREFIX = Constants.MARKER_PREFIX + "thread_";
+
     private static final String TEMPLATE = PROCESS_MARKER_PREFIX + "%s_processing";
-    
+
     @Private
     static final String INCOMING_PROCESS_MARKER_FILENAME = String.format(TEMPLATE, "incoming");
 
@@ -89,7 +90,7 @@ public final class DataMover
     private static final String[] PROCESS_MARKER_FILENAMES =
                 { INCOMING_PROCESS_MARKER_FILENAME, OUTGOING_PROCESS_MARKER_FILENAME,
                         LOCAL_PROCESS_MARKER_FILENAME, RECOVERY_PROCESS_MARKER_FILENAME };
-    
+
     private static final Logger operationLog =
             LogFactory.getLogger(LogCategory.OPERATION, DataMover.class);
 
@@ -149,8 +150,6 @@ public final class DataMover
     private final ITerminable start()
     {
         cleanUpProcessMarkerFiles();
-        File locationFile = new File(OUTGOING_TARGET_LOCATION_FILE);
-        FileUtilities.writeToFile(locationFile, parameters.getOutgoingTarget().getCanonicalPath());
         final DataMoverProcess outgoingProcess = createAndStartOutgoingProcess();
         final DataMoverProcess localProcess = createLocalProcess();
         final DataMoverProcess incomingProcess = createIncomingProcess();
@@ -161,8 +160,8 @@ public final class DataMover
         localProcess.startup(checkIntervalInternalMillis / 2L, checkIntervalInternalMillis);
         incomingProcess.startup(0L, parameters.getCheckIntervalMillis());
         // The ITerminable order here is important.
-        return new DataMoverTerminable(locationFile, recoveryProcess, incomingProcess,
-                localProcess, outgoingProcess);
+        return new DataMoverTerminable(recoveryProcess, incomingProcess, localProcess,
+                outgoingProcess);
     }
 
     private void cleanUpProcessMarkerFiles()

datamover/source/java/ch/systemsx/cisd/datamover/DataMoverShutdownHook.java

@@ -15,8 +15,13 @@
  */
 package ch.systemsx.cisd.datamover;
 
+import java.io.File;
+import java.io.IOException;
+
+import org.apache.commons.io.FileUtils;
 import org.apache.log4j.Logger;
 
+import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.utilities.ITerminable;
@@ -34,9 +39,38 @@ final class DataMoverShutdownHook implements ITriggerable
 
     private final ITerminable terminable;
 
-    DataMoverShutdownHook(final ITerminable terminable)
+    private final File outgoingTargetLocationFile;
+
+    DataMoverShutdownHook(final File locationFile, final ITerminable terminable)
     {
         this.terminable = terminable;
+        if (locationFile == null)
+        {
+            throw new IllegalArgumentException("Unspecified outgoing target location file.");
+        }
+        this.outgoingTargetLocationFile = locationFile;
+    }
+
+    private final static void createMarkerFile(final File markerFile)
+    {
+        try
+        {
+            FileUtils.touch(markerFile);
+        } catch (final IOException ex)
+        {
+            throw EnvironmentFailureException.fromTemplate(ex, "Can not create marker file '%s'.",
+                    markerFile.getAbsolutePath());
+        }
+    }
+
+    private final static void deleteFile(final File markerFile, final String description)
+    {
+        final boolean deleted = markerFile.delete();
+        if (deleted == false)
+        {
+            operationLog.warn(String.format("Can not delete %s file '%s'.", description, markerFile
+                    .getAbsolutePath()));
+        }
     }
 
     //
@@ -45,10 +79,14 @@ final class DataMoverShutdownHook implements ITriggerable
 
     public final void trigger()
     {
+        final File markerFile = new File(DataMover.SHUTDOWN_PROCESS_MARKER_FILENAME);
+        createMarkerFile(markerFile);
         if (operationLog.isInfoEnabled())
         {
             operationLog.info("Datamover is shutting down.");
         }
         terminable.terminate();
+        deleteFile(outgoingTargetLocationFile, "outgoing target location");
+        deleteFile(markerFile, "marker");
     }
 }
\ No newline at end of file

datamover/source/java/ch/systemsx/cisd/datamover/DataMoverTerminable.java

@@ -16,54 +16,29 @@
 
 package ch.systemsx.cisd.datamover;
 
-import java.io.File;
-import java.io.IOException;
+import org.apache.log4j.Logger;
 
-import org.apache.commons.io.FileUtils;
-
-import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.utilities.CompoundTerminable;
 import ch.systemsx.cisd.common.utilities.ITerminable;
 
 /**
- * The <i>DataMover</i> specific {@link ITerminable} implementation.
+ * The <i>DataMover</i> specific {@link CompoundTerminable} extension.
+ * <p>
+ * It try/catches each {@link #terminate(ITerminable)} call.
+ * </p>
  * 
  * @author Christian Ribeaud
  */
 final class DataMoverTerminable extends CompoundTerminable
 {
-    private final File outgoingTargetLocationFile;
+    private static final Logger operationLog =
+            LogFactory.getLogger(LogCategory.OPERATION, DataMoverTerminable.class);
 
-    DataMoverTerminable(final File locationFile, final DataMoverProcess... dataMoverProcesses)
+    DataMoverTerminable(final DataMoverProcess... dataMoverProcesses)
     {
         super(dataMoverProcesses);
-        if (locationFile == null)
-        {
-            throw new IllegalArgumentException("Unspecified outgoing target location file.");
-        }
-        this.outgoingTargetLocationFile = locationFile;
-    }
-
-    private final static void createMarkerFile(final File markerFile)
-    {
-        try
-        {
-            FileUtils.touch(markerFile);
-        } catch (final IOException ex)
-        {
-            throw EnvironmentFailureException.fromTemplate(ex, "Can not create marker file '%s'.",
-                    markerFile.getAbsolutePath());
-        }
-    }
-
-    private final static void deleteMarkerFile(final File markerFile)
-    {
-        final boolean deleted = markerFile.delete();
-        if (deleted == false)
-        {
-            throw EnvironmentFailureException.fromTemplate("Can not delete marker file '%s'.",
-                    markerFile.getAbsolutePath());
-        }
     }
 
     //
@@ -71,13 +46,16 @@ final class DataMoverTerminable extends CompoundTerminable
     //
 
     @Override
-    public final boolean terminate()
+    protected final boolean terminate(final ITerminable terminable)
     {
-        final File markerFile = new File(DataMover.SHUTDOWN_PROCESS_MARKER_FILENAME);
-        createMarkerFile(markerFile);
-        final boolean terminate = super.terminate();
-        deleteMarkerFile(markerFile);
-        outgoingTargetLocationFile.delete();
-        return terminate;
+        try
+        {
+            return super.terminate(terminable);
+        } catch (final Exception e)
+        {
+            operationLog.warn(String
+                    .format("Terminating Datamover process '%s' threw an exception."), e);
+            return false;
+        }
     }
 }

datamover/source/java/ch/systemsx/cisd/datamover/Main.java

@@ -29,6 +29,7 @@ import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.logging.LogInitializer;
 import ch.systemsx.cisd.common.utilities.BuildAndEnvironmentInfo;
+import ch.systemsx.cisd.common.utilities.FileUtilities;
 import ch.systemsx.cisd.common.utilities.ITerminable;
 import ch.systemsx.cisd.common.utilities.TriggeringTimerTask;
 import ch.systemsx.cisd.datamover.filesystem.FileStoreFactory;
@@ -45,6 +46,7 @@ import ch.systemsx.cisd.datamover.utils.LocalBufferDirs;
  */
 public final class Main
 {
+
     private static final Logger operationLog =
             LogFactory.getLogger(LogCategory.OPERATION, Main.class);
 
@@ -130,11 +132,12 @@ public final class Main
         }
     }
 
-    private final static void createShutdownHookTimer(final ITerminable terminable)
+    private final static void createShutdownHookTimer(final File outgoingTargetLocationFile,
+            final ITerminable terminable)
     {
         final TriggeringTimerTask shutdownHook =
                 new TriggeringTimerTask(new File(DataMover.SHUTDOWN_MARKER_FILENAME),
-                        new DataMoverShutdownHook(terminable));
+                        new DataMoverShutdownHook(outgoingTargetLocationFile, terminable));
         new Timer("Shutdown Hook", true).schedule(shutdownHook, 0L, 5000L);
     }
 
@@ -148,8 +151,11 @@ public final class Main
     private static void startupServer(final Parameters parameters)
     {
         final IFileSysOperationsFactory factory = new FileSysOperationsFactory(parameters);
+        final File outgoingTargetLocationFile = new File(DataMover.OUTGOING_TARGET_LOCATION_FILE);
+        FileUtilities.writeToFile(outgoingTargetLocationFile, parameters.getOutgoingTarget()
+                .getCanonicalPath());
         final ITerminable terminable = DataMover.start(parameters, factory);
-        createShutdownHookTimer(terminable);
+        createShutdownHookTimer(outgoingTargetLocationFile, terminable);
     }
 
     public static void main(final String[] args)