Add JMX memory monitoring Spring bean to both application server and datastore server.

SVN: 26557

common/source/java/ch/systemsx/cisd/common/jmx/JMXMemoryMonitor.java

@@ -44,7 +44,7 @@ public class JMXMemoryMonitor
     private static final Logger notifyLog = LogFactory.getLogger(LogCategory.NOTIFY,
             JMXMemoryMonitor.class);
 
-    private final long logIntervallMillis;
+    private final long logIntervalMillis;
 
     private final int memoryHighwaterMarkPercent;
 
@@ -57,24 +57,24 @@ public class JMXMemoryMonitor
     /**
      * Starts the memory monitor.
      * 
-     * @param monitoringIntervallMillis Interval (in ms) for monitoring memory consumption.
-     * @param logIntervallMillis Interval (in ms) for regular logging of memory consumption.
+     * @param monitoringIntervalMillis Interval (in ms) for monitoring memory consumption.
+     * @param logIntervalMillis Interval (in ms) for regular logging of memory consumption.
      * @param memoryHighWatermarkPercent High-water mark for heap and non-heap memory consumption
      *            (in percent of the maximal memory). If this mark is exceeded, a warning
      *            notification will be sent.
      */
-    public static final void startMonitor(long monitoringIntervallMillis, long logIntervallMillis,
+    public static final void startMonitor(long monitoringIntervalMillis, long logIntervalMillis,
             int memoryHighWatermarkPercent)
     {
-        final Timer timer = new Timer(true);
+        final Timer timer = new Timer("memory monitor", true);
         final JMXMemoryMonitor monitor =
-                new JMXMemoryMonitor(logIntervallMillis, memoryHighWatermarkPercent);
-        timer.schedule(monitor.getTimerTask(), 0L, monitoringIntervallMillis);
+                new JMXMemoryMonitor(logIntervalMillis, memoryHighWatermarkPercent);
+        timer.schedule(monitor.getTimerTask(), 0L, monitoringIntervalMillis);
     }
 
     JMXMemoryMonitor(long logIntervallMillis, int memoryHighWatermarkPercent)
     {
-        this.logIntervallMillis = logIntervallMillis;
+        this.logIntervalMillis = logIntervallMillis;
         this.memoryHighwaterMarkPercent = memoryHighWatermarkPercent;
     }
 
@@ -125,7 +125,7 @@ public class JMXMemoryMonitor
                         nonHeapMemoryExhaustionNotificationSent = false;
                     }
                     final long timeSinceLastLoggedMillis = now - lastLoggedMillis;
-                    if (timeSinceLastLoggedMillis > logIntervallMillis)
+                    if (logIntervalMillis >= 0 && timeSinceLastLoggedMillis > logIntervalMillis)
                     {
                         machineLog.info(String.format(
                                 "Heap memory used: %.1f GB, non-heap memory used: %.1f GB", mbean

common/source/java/ch/systemsx/cisd/common/jmx/JMXMemoryMonitorSprintBean.java

+/*
+ * Copyright 2012 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.common.jmx;
+
+import java.util.Properties;
+
+import javax.annotation.Resource;
+
+import org.apache.log4j.Logger;
+import org.springframework.beans.factory.InitializingBean;
+
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.spring.ExposablePropertyPlaceholderConfigurer;
+
+/**
+ * A starter for {@link JMXMemoryMonitor}. Can be used to start the memory monitor as a Spring bean.
+ * 
+ * @author Bernd Rinn
+ */
+public class JMXMemoryMonitorSprintBean implements InitializingBean
+{
+    private static final String MEMORY_MONITORING_INTERVAL_PROP =
+            "memorymonitor-monitoring-interval";
+
+    private static final String MEMORY_MONITORING_LOG_INTERVAL_PROP = "memorymonitor-log-interval";
+
+    private static final String MEMORY_MONITORING_HIGH_WATERMARK_PERCENT_PROP =
+            "memorymonitor-high-watermark-percent";
+
+    private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+            JMXMemoryMonitorSprintBean.class);
+
+    private static final long MILLISECONDS = 1000L;
+
+    private final static long DEFAULT_MONITORING_INTERVALL_MILLIS = 60 * MILLISECONDS;
+
+    private final static long DEFAULT_LOG_INTERVAL_MILLIS =
+            60 * DEFAULT_MONITORING_INTERVALL_MILLIS;
+
+    private final static int DEFAULT_MEMORY_HIGH_WATERMARK_PERCENT = 90;
+
+    @Resource(name = ExposablePropertyPlaceholderConfigurer.PROPERTY_CONFIGURER_BEAN_NAME)
+    private ExposablePropertyPlaceholderConfigurer configurer;
+
+    @Override
+    public void afterPropertiesSet() throws Exception
+    {
+        final Properties props = configurer.getResolvedProps();
+        final long monitoringIntervalMillis =
+                (props.getProperty(MEMORY_MONITORING_INTERVAL_PROP) == null) ? DEFAULT_MONITORING_INTERVALL_MILLIS
+                        : Integer.parseInt(props.getProperty(MEMORY_MONITORING_INTERVAL_PROP))
+                                * MILLISECONDS;
+        final long logIntervalMillis =
+                (props.getProperty(MEMORY_MONITORING_LOG_INTERVAL_PROP) == null) ? DEFAULT_LOG_INTERVAL_MILLIS
+                        : Integer.parseInt(props.getProperty(MEMORY_MONITORING_LOG_INTERVAL_PROP))
+                                * MILLISECONDS;
+        final int memoryHighWatermarkPercent =
+                (props.getProperty(MEMORY_MONITORING_HIGH_WATERMARK_PERCENT_PROP) == null) ? DEFAULT_MEMORY_HIGH_WATERMARK_PERCENT
+                        : Integer.parseInt(props
+                                .getProperty(MEMORY_MONITORING_HIGH_WATERMARK_PERCENT_PROP));
+        if (monitoringIntervalMillis > 0)
+        {
+            if (operationLog.isInfoEnabled())
+            {
+                operationLog
+                        .info(String
+                                .format("Starting JMX Memory monitor with parameters %s=%s, %s=%s, %s=%s",
+                                        MEMORY_MONITORING_INTERVAL_PROP,
+                                        monitoringIntervalMillis / 1000,
+                                        MEMORY_MONITORING_LOG_INTERVAL_PROP,
+                                        (logIntervalMillis < 0) ? "DISABLED"
+                                                : logIntervalMillis / 1000,
+                                        MEMORY_MONITORING_HIGH_WATERMARK_PERCENT_PROP,
+                                        (memoryHighWatermarkPercent >= 100) ? "DISABLED"
+                                                : memoryHighWatermarkPercent));
+            }
+            JMXMemoryMonitor.startMonitor(monitoringIntervalMillis, logIntervalMillis,
+                    memoryHighWatermarkPercent);
+        }
+    }
+
+}

datastore_server/dist/etc/service.properties

@@ -76,6 +76,20 @@ username = etlserver
 # The password to use when contacting the openBIS server
 password = etlserver
 
+#
+# JMX memory monitor
+#
+
+# Interval between two runs of the memory monitor (in seconds). 
+# Set to -1 to disable the memory monitor.
+memorymonitor-monitoring-interval = 60
+# Interval between two regular log call of the memory monitor (in seconds).
+# Set to -1 to disable regular memory usage logging. 
+memorymonitor-log-interval = 3600
+# The percentage of memory that, if exceeded, triggers a notify log of the memory manager, 
+# Set to 100 to disable.
+memorymonitor-high-watermark-percent = 90
+
 #
 # CIFEX configuration: Only needed if data export should work without the user having to type in 
 # his password. Note that in in order for this to work the CIFEX server needs to be configured to 

datastore_server/source/java/dssApplicationContext.xml

@@ -27,42 +27,44 @@
         factory-method="getResolvedProps">
     </bean>
     
+	<bean id="memory-monitor" class="ch.systemsx.cisd.common.jmx.JMXMemoryMonitorSprintBean" />
+
     <bean id="data-set-path-infos-provider" class="ch.systemsx.cisd.openbis.dss.generic.server.DatabaseBasedDataSetPathInfoProvider"/>
 
-  <bean id="plugin-tasks" class="ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.PluginTaskInfoProvider"
+	  <bean id="plugin-tasks" class="ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.PluginTaskInfoProvider"
         factory-method="create" />
 
-  <bean id="etl-lims-service" class="ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService"
-      factory-method="createOpenBisService">
-      <constructor-arg value="${server-url}"/>
-      <constructor-arg value="${server-timeout-in-minutes}"/>
-  </bean>
-  
-  <bean id="general-information-service" class="ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService"
-      factory-method="createGeneralInformationService">
-      <constructor-arg value="${server-url}"/>
-      <constructor-arg value="${server-timeout-in-minutes}"/>
-  </bean>
-  
-  <bean id="sessionHolder" class="ch.systemsx.cisd.openbis.generic.shared.dto.OpenBISSessionHolder">
-      <property name="dataStoreCode" value="${data-store-server-code}"/>
-  </bean>
-  
-  <bean id="reauthenticateInterceptor" class="ch.systemsx.cisd.openbis.dss.generic.server.openbisauth.OpenBISAuthenticationInterceptor">
-        <constructor-arg ref="session-token-manager"/>
-        <constructor-arg ref="etl-lims-service"/>
-        <constructor-arg ref="plugin-tasks" />
-        <constructor-arg ref="sessionHolder" />
-        <property name="username" value="${username}"/>
-        <property name="password" value="${password}"/>
-        <property name="port" value="${port}"/>
-        <property name="useSSL" value="${use-ssl}"/>
-        <property name="downloadUrl" value="${download-url}"/>
-  </bean>
+	  <bean id="etl-lims-service" class="ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService"
+	      factory-method="createOpenBisService">
+	      <constructor-arg value="${server-url}"/>
+	      <constructor-arg value="${server-timeout-in-minutes}"/>
+	  </bean>
   
-  <bean id="reauthenticateAdvisor" class="ch.systemsx.cisd.openbis.dss.generic.server.openbisauth.OpenBISAuthenticationAdvisor">
-      <constructor-arg ref="reauthenticateInterceptor"/>
-  </bean>
+	  <bean id="general-information-service" class="ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService"
+	      factory-method="createGeneralInformationService">
+	      <constructor-arg value="${server-url}"/>
+	      <constructor-arg value="${server-timeout-in-minutes}"/>
+	  </bean>
+	  
+	  <bean id="sessionHolder" class="ch.systemsx.cisd.openbis.generic.shared.dto.OpenBISSessionHolder">
+	      <property name="dataStoreCode" value="${data-store-server-code}"/>
+	  </bean>
+	  
+	  <bean id="reauthenticateInterceptor" class="ch.systemsx.cisd.openbis.dss.generic.server.openbisauth.OpenBISAuthenticationInterceptor">
+	        <constructor-arg ref="session-token-manager"/>
+	        <constructor-arg ref="etl-lims-service"/>
+	        <constructor-arg ref="plugin-tasks" />
+	        <constructor-arg ref="sessionHolder" />
+	        <property name="username" value="${username}"/>
+	        <property name="password" value="${password}"/>
+	        <property name="port" value="${port}"/>
+	        <property name="useSSL" value="${use-ssl}"/>
+	        <property name="downloadUrl" value="${download-url}"/>
+	  </bean>
+	  
+	  <bean id="reauthenticateAdvisor" class="ch.systemsx.cisd.openbis.dss.generic.server.openbisauth.OpenBISAuthenticationAdvisor">
+	      <constructor-arg ref="reauthenticateInterceptor"/>
+	  </bean>
    
     <bean id="openBIS-service" class="ch.systemsx.cisd.openbis.dss.generic.server.EncapsulatedOpenBISService">
        <constructor-arg ref="etl-lims-service"/>

openbis/dist/server/service.properties

@@ -77,6 +77,20 @@ ldap.queryTemplate =
 # The database instance local unique identifier. Used when the new database is created.
 database-instance = TEST
 
+#
+# JMX memory monitor
+#
+
+# Interval between two runs of the memory monitor (in seconds). 
+# Set to -1 to disable the memory monitor.
+memorymonitor-monitoring-interval = 60
+# Interval between two regular log call of the memory monitor (in seconds).
+# Set to -1 to disable regular memory usage logging. 
+memorymonitor-log-interval = 3600
+# The percentage of memory that, if exceeded, triggers a notify log of the memory manager, 
+# Set to 100 to disable.
+memorymonitor-high-watermark-percent = 90
+
 # The URL of the CIFEX server
 cifex-url = https://cifex.ethz.ch:443
 

openbis/source/java/genericApplicationContext.xml

@@ -17,6 +17,8 @@
 
     <bean id="exception-translator" class="ch.systemsx.cisd.openbis.generic.server.business.bo.NextExceptionFallbackExceptionTranslator" />
 
+	<bean id="memory-monitor" class="ch.systemsx.cisd.common.jmx.JMXMemoryMonitorSprintBean" />
+
     <bean id="transaction-manager"
         class="ch.systemsx.cisd.openbis.generic.server.dataaccess.db.OpenBISHibernateTransactionManager">
         <constructor-arg ref="dao-factory" />