[LMS-2583] new IFreeSpaceProvider implementation respecting the free space...

[LMS-2583] new IFreeSpaceProvider implementation respecting the free space within a PostgreSQL data source,
IFreespaceProvider is now configurable in ExperimentBaseArchivingTask

SVN: 23403

datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTask.java

@@ -40,6 +40,8 @@ import ch.systemsx.cisd.common.filesystem.SimpleFreeSpaceProvider;
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.maintenance.IDataStoreLockingMaintenanceTask;
+import ch.systemsx.cisd.common.utilities.ClassUtils;
+import ch.systemsx.cisd.common.utilities.ExtendedProperties;
 import ch.systemsx.cisd.common.utilities.PropertyParametersUtil;
 import ch.systemsx.cisd.common.utilities.PropertyUtils;
 import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService;
@@ -77,12 +79,14 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
 
     static final String MONITORED_DIR = "monitored-dir";
 
+    static final String FREE_SPACE_PROVIDER_PREFIX = "free-space-provider.";
+
     private static final EnumSet<DataSetArchivingStatus> ARCHIVE_STATES = EnumSet.of(
             DataSetArchivingStatus.ARCHIVE_PENDING, DataSetArchivingStatus.ARCHIVED);
 
     private final IEncapsulatedOpenBISService service;
 
-    private final IFreeSpaceProvider freeSpaceProvider;
+    private IFreeSpaceProvider freeSpaceProvider;
 
     private File storeRoot;
 
@@ -98,15 +102,12 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
 
     public ExperimentBasedArchivingTask()
     {
-        this(ServiceProvider.getOpenBISService(), new SimpleFreeSpaceProvider(), ServiceProvider
-                .getShareIdManager());
+        this(ServiceProvider.getOpenBISService(), ServiceProvider.getShareIdManager());
     }
 
-    ExperimentBasedArchivingTask(IEncapsulatedOpenBISService service,
-            IFreeSpaceProvider freeSpaceProvider, IShareIdManager shareIdManager)
+    ExperimentBasedArchivingTask(IEncapsulatedOpenBISService service, IShareIdManager shareIdManager)
     {
         this.service = service;
-        this.freeSpaceProvider = freeSpaceProvider;
         this.shareIdManager = shareIdManager;
     }
 
@@ -116,15 +117,33 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
     }
 
     public void setUp(String pluginName, Properties properties)
+    {
+        storeRoot = setUpStoreRoot(properties);
+        freeSpaceProvider = setUpFreeSpaceProvider(properties);
+        setUpMonitoredShareOrPath(properties);
+        minimumFreeSpace =
+                FileUtils.ONE_MB * PropertyUtils.getLong(properties, MINIMUM_FREE_SPACE_KEY, 1024);
+        excludedDataSetTypes =
+                new HashSet<String>(Arrays.asList(PropertyParametersUtil.parseItemisedProperty(
+                        properties.getProperty(EXCLUDED_DATA_SET_TYPES_KEY, ""),
+                        EXCLUDED_DATA_SET_TYPES_KEY)));
+    }
+
+    private File setUpStoreRoot(Properties properties)
     {
         String storeRootFileName =
                 PropertyUtils.getMandatoryProperty(properties, STOREROOT_DIR_KEY);
-        storeRoot = new File(storeRootFileName);
-        if (storeRoot.isDirectory() == false)
+        File resultStoreRoot = new File(storeRootFileName);
+        if (resultStoreRoot.isDirectory() == false)
         {
             throw new ConfigurationFailureException(
                     "Store root doesn't exists or isn't a directory: " + storeRoot);
         }
+        return resultStoreRoot;
+    }
+
+    private void setUpMonitoredShareOrPath(Properties properties)
+    {
         final String monitoredDirPath = PropertyUtils.getProperty(properties, MONITORED_DIR);
         if (monitoredDirPath == null)
         {
@@ -147,12 +166,35 @@ public class ExperimentBasedArchivingTask implements IDataStoreLockingMaintenanc
                         + "' doesn't exists or isn't a directory.");
             }
         }
-        minimumFreeSpace =
-                FileUtils.ONE_MB * PropertyUtils.getLong(properties, MINIMUM_FREE_SPACE_KEY, 1024);
-        excludedDataSetTypes =
-                new HashSet<String>(Arrays.asList(PropertyParametersUtil.parseItemisedProperty(
-                        properties.getProperty(EXCLUDED_DATA_SET_TYPES_KEY, ""),
-                        EXCLUDED_DATA_SET_TYPES_KEY)));
+    }
+
+    private IFreeSpaceProvider setUpFreeSpaceProvider(Properties properties)
+    {
+        Properties providerProps =  ExtendedProperties.getSubset(properties, FREE_SPACE_PROVIDER_PREFIX, true);
+        String freeSpaceProviderClassName =
+                PropertyUtils.getProperty(providerProps, "class",
+                        SimpleFreeSpaceProvider.class.getName());
+        
+        Class<?> clazz = null;
+        try
+        {
+            clazz = Class.forName(freeSpaceProviderClassName);
+        } catch (ClassNotFoundException cnfe)
+        {
+            throw ConfigurationFailureException.fromTemplate(
+                    "Cannot find configured free space provider class '%s'",
+                    freeSpaceProviderClassName);
+        }
+        
+        if (ClassUtils.hasConstructor(clazz, properties))
+        {
+            return ClassUtils.create(IFreeSpaceProvider.class, clazz, properties);
+
+        } else
+        {
+            return ClassUtils.create(IFreeSpaceProvider.class, clazz);
+        }
+        
     }
 
     public void execute()

datastore_server/source/java/ch/systemsx/cisd/etlserver/registrator/api/v1/impl/DataSourceQueryService.java

@@ -28,7 +28,7 @@ import org.apache.log4j.Logger;
 
 import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
-import ch.systemsx.cisd.openbis.dss.generic.shared.DataSourceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSourceProvider;
 import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
 import ch.systemsx.cisd.openbis.dss.generic.shared.api.internal.v1.IDataSourceQueryService;
 
@@ -40,7 +40,7 @@ public class DataSourceQueryService implements IDataSourceQueryService
     private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
             DataSourceQueryService.class);
 
-    private DataSourceProvider getDataSourceProvider()
+    private IDataSourceProvider getDataSourceProvider()
     {
         return ServiceProvider.getDataSourceProvider();
     }

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/DataSourceProvider.java

@@ -29,8 +29,8 @@ import ch.systemsx.cisd.common.logging.LogCategory;
 import ch.systemsx.cisd.common.logging.LogFactory;
 import ch.systemsx.cisd.common.utilities.ClassUtils;
 import ch.systemsx.cisd.common.utilities.PropertyParametersUtil;
-import ch.systemsx.cisd.common.utilities.PropertyUtils;
 import ch.systemsx.cisd.common.utilities.PropertyParametersUtil.SectionProperties;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
 import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUtil;
 
 /**
@@ -68,7 +68,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUt
  * 
  * @author Izabela Adamczyk
  */
-public class DataSourceProvider
+public class DataSourceProvider implements IDataSourceProvider
 {
     static final String DATA_SOURCES_KEY = "data-sources";
 

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataSourceProvider.java

+/*
+ * Copyright 2010 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.shared;
+
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+/**
+ * A provider for data sources.
+ * 
+ * @author Kaloyan Enimanev
+ */
+public interface IDataSourceProvider
+{
+
+    /**
+     * Returns data source configured with given name or throws {@link IllegalArgumentException} if
+     * not configured.
+     */
+    public DataSource getDataSource(String name);
+
+    /**
+     * Extracts the data source name from the specified properties and returns the requested data
+     * source by calling {@link #getDataSource(String)}.
+     */
+    public DataSource getDataSource(Properties properties);
+
+}

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/ServiceProvider.java

@@ -150,9 +150,9 @@ public class ServiceProvider
                 "data-store-rpc-service-generic"));
     }
 
-    public static DataSourceProvider getDataSourceProvider()
+    public static IDataSourceProvider getDataSourceProvider()
     {
-        return ((DataSourceProvider) getApplicationContext().getBean("data-source-provider"));
+        return ((IDataSourceProvider) getApplicationContext().getBean("data-source-provider"));
     }
 
     public static IConfigProvider getConfigProvider()

datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/PostgresPlusFileSystemFreeSpaceProvider.java

+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.shared.utils;
+
+import java.io.IOException;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.SQLException;
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.apache.log4j.Logger;
+
+import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel;
+import ch.systemsx.cisd.common.filesystem.HostAwareFile;
+import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider;
+import ch.systemsx.cisd.common.filesystem.SimpleFreeSpaceProvider;
+import ch.systemsx.cisd.common.logging.LogCategory;
+import ch.systemsx.cisd.common.logging.LogFactory;
+import ch.systemsx.cisd.common.utilities.PropertyUtils;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider;
+
+/**
+ * PostgreSQL database files are designed to grow until they take up the entire available disk
+ * space. It is therefore impossible to estimate what part of a hard disk is "free" by just asking
+ * the file system.
+ * <p>
+ * The {@link PostgresPlusFileSystemFreeSpaceProvider} estimates the free space on a drive as the
+ * sum of the free disk space and the free space of a PostgreSQL database as returned by its
+ * "pgstattuple" extension.
+ * 
+ * <pre>
+ * IMPORTANT: The class requires that the extension 'pgstattuple' is installed on the target
+ * PostgreSQL database. For PostgreSQL 9.1 this can be done by executing :
+ *     
+ *     psql -d DB_NAME -c "CREATE EXTENSION pgstattuple;"
+ * 
+ * @author Kaloyan Enimanev
+ */
+public class PostgresPlusFileSystemFreeSpaceProvider implements IFreeSpaceProvider
+{
+    
+    static final String EXECUTE_VACUUM_KEY = "execute-vacuum";
+
+    static final String DATA_SOURCE_KEY = "monitored-data-source";
+
+    private static final String VACUUM_QUERY = "VACUUM;";
+
+    private static final String CREATE_TMP_FREE_SPACE_TABLE =
+            "CREATE TEMPORARY TABLE freespace ON COMMIT DROP AS "
+                    + "   (SELECT relname, (SELECT free_space FROM pgstattuple(relid)) "
+                    + "       AS free_space FROM pg_catalog.pg_statio_user_tables);";
+
+    private static final String SELECT_FREE_SPACE_QUERY = CREATE_TMP_FREE_SPACE_TABLE
+            + " SELECT sum(free_space) FROM freespace;";
+
+    private static final Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION,
+            PostgresPlusFileSystemFreeSpaceProvider.class);
+
+    private final boolean executeVacuum;
+
+    private final DataSource dataSource;
+
+    private final IFreeSpaceProvider fileSystemFreeSpaceProvider = new SimpleFreeSpaceProvider();
+
+    public PostgresPlusFileSystemFreeSpaceProvider(Properties properties)
+    {
+        executeVacuum = PropertyUtils.getBoolean(properties, EXECUTE_VACUUM_KEY, false);
+        
+        String dataSourceName = PropertyUtils.getMandatoryProperty(properties, DATA_SOURCE_KEY);
+        dataSource = ServiceProvider.getDataSourceProvider().getDataSource(dataSourceName);
+    }
+
+    public long freeSpaceKb(HostAwareFile path) throws IOException
+    {
+        long dataSourceFreeSpace = calculateDataSourceFreeSpace();
+        long fsFreeSpace = fileSystemFreeSpaceProvider.freeSpaceKb(path);
+        return dataSourceFreeSpace + fsFreeSpace;
+    }
+
+    private long calculateDataSourceFreeSpace()
+    {
+        Connection connection = null;
+        try
+        {
+            connection = createConnection();
+            if (executeVacuum)
+            {
+                executeVacuumQuery(connection);
+            }
+
+            return calculateFreeSpace(connection);
+
+        } catch (SQLException sqlEx)
+        {
+            throw CheckedExceptionTunnel.wrapIfNecessary(sqlEx);
+        } finally
+        {
+            closeConnection(connection);
+        }
+    }
+
+    private void executeVacuumQuery(Connection connection) throws SQLException
+    {
+        connection.createStatement().execute(VACUUM_QUERY);
+    }
+
+    private long calculateFreeSpace(Connection connection) throws SQLException
+    {
+        ResultSet result = connection.createStatement().executeQuery(SELECT_FREE_SPACE_QUERY);
+        return result.getLong(1);
+    }
+
+    private Connection createConnection() throws SQLException
+    {
+        return dataSource.getConnection();
+    }
+
+    private void closeConnection(Connection connection)
+    {
+        if (connection != null)
+        {
+            try
+            {
+                connection.close();
+            } catch (SQLException ex)
+            {
+                // suppress this exception
+                operationLog.error(ex);
+            }
+        }
+    }
+
+}

datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/plugins/ExperimentBasedArchivingTaskTest.java

@@ -54,6 +54,17 @@ import ch.systemsx.cisd.openbis.generic.shared.dto.identifier.ProjectIdentifier;
  */
 public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
 {
+    public static class MockFreeSpaceProvider implements IFreeSpaceProvider
+    {
+
+        static IFreeSpaceProvider mock;
+
+        public long freeSpaceKb(HostAwareFile path) throws IOException
+        {
+            return mock.freeSpaceKb(path);
+        }
+    }
+
     private static final String LOG_ENTRY_PREFIX_TEMPLATE =
             "INFO  %s.ExperimentBasedArchivingTask - ";
 
@@ -121,7 +132,9 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
         service = context.mock(IEncapsulatedOpenBISService.class);
         freeSpaceProvider = context.mock(IFreeSpaceProvider.class);
         shareIdManager = context.mock(IShareIdManager.class);
-        task = new ExperimentBasedArchivingTask(service, freeSpaceProvider, shareIdManager);
+        MockFreeSpaceProvider.mock = freeSpaceProvider;
+
+        task = new ExperimentBasedArchivingTask(service, shareIdManager);
         assertEquals(true, task.requiresDataStoreLock());
 
         e1 = new ExperimentBuilder().id(41).identifier("/S/P/E1").getExperiment();
@@ -149,6 +162,8 @@ public class ExperimentBasedArchivingTaskTest extends AbstractFileSystemTestCase
         properties.setProperty(ExperimentBasedArchivingTask.MONITORED_SHARE_KEY, SHARE_ID);
         properties.setProperty(ExperimentBasedArchivingTask.MINIMUM_FREE_SPACE_KEY, "100");
         properties.setProperty(ExperimentBasedArchivingTask.EXCLUDED_DATA_SET_TYPES_KEY, "ABC, B");
+        properties.setProperty(ExperimentBasedArchivingTask.FREE_SPACE_PROVIDER_PREFIX + "class",
+                MockFreeSpaceProvider.class.getName());
     }
 
     private DataSetBuilder dataSet(String code)

datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/ServiceProviderTestWrapper.java

@@ -43,6 +43,7 @@ public class ServiceProviderTestWrapper
         classNameToBeanName.put(IEncapsulatedOpenBISService.class, "openBIS-service");
         classNameToBeanName.put(IShareIdManager.class, "share-id-manager");
         classNameToBeanName.put(IConfigProvider.class, "config-provider");
+        classNameToBeanName.put(IDataSourceProvider.class, "data-source-provider");
         classNameToBeanName
                 .put(IHierarchicalContentProvider.class, "hierarchical-content-provider");
     }

datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/PostgresPlusFileSystemFreeSpaceProviderTest.java

+/*
+ * Copyright 2011 ETH Zuerich, CISD
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package ch.systemsx.cisd.openbis.dss.generic.shared.utils;
+
+import static ch.systemsx.cisd.openbis.dss.generic.shared.utils.PostgresPlusFileSystemFreeSpaceProvider.DATA_SOURCE_KEY;
+
+import java.io.File;
+import java.sql.Connection;
+import java.sql.ResultSet;
+import java.sql.Statement;
+import java.util.Properties;
+
+import javax.sql.DataSource;
+
+import org.apache.commons.io.FileSystemUtils;
+import org.jmock.Expectations;
+import org.jmock.Mockery;
+import org.springframework.beans.factory.BeanFactory;
+import org.testng.AssertJUnit;
+import org.testng.annotations.AfterMethod;
+import org.testng.annotations.BeforeMethod;
+import org.testng.annotations.Test;
+
+import ch.systemsx.cisd.common.filesystem.HostAwareFile;
+import ch.systemsx.cisd.openbis.dss.generic.shared.IDataSourceProvider;
+import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProviderTestWrapper;
+
+/**
+ * @author Kaloyan Enimanev
+ */
+public class PostgresPlusFileSystemFreeSpaceProviderTest extends AssertJUnit
+{
+
+    private final static String DATA_SOURCE = "data-source";
+
+    private PostgresPlusFileSystemFreeSpaceProvider provider;
+
+    private Mockery context;
+    private Connection connection;
+
+    private BeanFactory mockApplicationContext;
+
+    @BeforeMethod
+    public void setUp() throws Exception
+    {
+        context = new Mockery();
+        mockApplicationContext = context.mock(BeanFactory.class);
+        connection = context.mock(Connection.class);
+
+        ServiceProviderTestWrapper.setApplicationContext(mockApplicationContext);
+        
+        final IDataSourceProvider dsProvider =
+                ServiceProviderTestWrapper.mock(context, IDataSourceProvider.class);
+        final DataSource dataSource = context.mock(DataSource.class);
+
+        context.checking(new Expectations()
+            {
+                {
+                    allowing(dsProvider).getDataSource(DATA_SOURCE);
+                    will(returnValue(dataSource));
+
+                    one(dataSource).getConnection();
+                    will(returnValue(connection));
+
+                    one(connection).close();
+                }
+            });
+    }
+
+    @AfterMethod
+    public void tearDown()
+    {
+        ServiceProviderTestWrapper.restoreApplicationContext();
+    }
+
+    private Properties createProperties(boolean executeVacuum) {
+        Properties props = new Properties();
+        props.put(DATA_SOURCE_KEY, DATA_SOURCE);
+        if (executeVacuum)
+        {
+            props.put(PostgresPlusFileSystemFreeSpaceProvider.EXECUTE_VACUUM_KEY, "true");
+        }
+        return props;
+    }
+
+    @Test
+    public void testNoVacuum() throws Exception
+    {
+        Properties props = createProperties(false);
+        provider = new PostgresPlusFileSystemFreeSpaceProvider(props);
+
+        final long postgresFreeSpace = 1000L;
+        prepareFreeSpaceExpectations(postgresFreeSpace);
+        
+        File workDir = new File(".");
+        HostAwareFile file = new HostAwareFile(workDir);
+        long fsFreeSpace = FileSystemUtils.freeSpaceKb(workDir.getAbsolutePath());
+        long totalFreeSpace = provider.freeSpaceKb(file);
+        
+        assertEquals(fsFreeSpace + postgresFreeSpace, totalFreeSpace);
+    }
+
+    @Test
+    public void testWithVacuum() throws Exception
+    {
+        Properties props = createProperties(true);
+        provider = new PostgresPlusFileSystemFreeSpaceProvider(props);
+
+        final long postgresFreeSpace = 1000L;
+        prepareVacuumExpectations();
+        prepareFreeSpaceExpectations(postgresFreeSpace);
+
+        File workDir = new File(".");
+        HostAwareFile file = new HostAwareFile(workDir);
+        long fsFreeSpace = FileSystemUtils.freeSpaceKb(workDir.getAbsolutePath());
+        long totalFreeSpace = provider.freeSpaceKb(file);
+
+        assertEquals(fsFreeSpace + postgresFreeSpace, totalFreeSpace);
+    }
+
+    private void prepareVacuumExpectations() throws Exception
+    {
+        context.checking(new Expectations()
+            {
+                {
+                    Statement statement = context.mock(Statement.class, "vacuumStatement");
+                    one(connection).createStatement();
+                    will(returnValue(statement));
+
+                    one(statement).execute("VACUUM;");
+                }
+            });
+    }
+
+    private void prepareFreeSpaceExpectations(final long freeSpace) throws Exception
+    {
+        context.checking(new Expectations()
+            {
+                {
+                    Statement statement = context.mock(Statement.class, "freeSpaceStatement");
+                    one(connection).createStatement();
+                    will(returnValue(statement));
+
+                    one(statement).executeQuery(with(any(String.class)));
+
+                    ResultSet rs = context.mock(ResultSet.class);
+                    will(returnValue(rs));
+
+                    one(rs).getLong(1);
+                    will(returnValue(freeSpace));
+                }
+            });
+    }
+
+}