From 4546b79b0ef1f73157a7bfd5740d7c886544eb05 Mon Sep 17 00:00:00 2001 From: felmer <felmer> Date: Thu, 10 Feb 2011 10:11:41 +0000 Subject: [PATCH] LMS-1995 data store code added to SimpleDataSetInformationDTO, SegementedStoreBalanceingTask and ISegmentedStoreBalancer introduced. SVN: 19869 --- .../plugins/ISegmentedStoreBalancer.java | 34 +++++ .../cisd/etlserver/plugins/NonBalancer.java | 65 +++++++++ .../plugins/SegmentedStoreBalancingTask.java | 131 ++++++++++++++++++ .../utils/DssPropertyParametersUtil.java | 5 +- .../shared/utils/SegmentedStoreUtils.java | 81 ++++++++++- .../dss/generic/shared/utils/Share.java | 118 ++++++++++++++++ .../business/bo/SimpleDataSetHelper.java | 1 + .../dto/SimpleDataSetInformationDTO.java | 12 ++ 8 files changed, 441 insertions(+), 6 deletions(-) create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreBalancer.java create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/NonBalancer.java create mode 100644 datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTask.java create mode 100644 datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreBalancer.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreBalancer.java new file mode 100644 index 00000000000..731c6a61c47 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/ISegmentedStoreBalancer.java @@ -0,0 +1,34 @@ +/* + * Copyright 2011 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.etlserver.plugins; + +import java.util.List; + +import ch.systemsx.cisd.common.logging.ISimpleLogger; +import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; +import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share; + +/** + * + * + * @author Franz-Josef Elmer + */ +public interface ISegmentedStoreBalancer +{ + public void balanceStore(List<Share> shares, IEncapsulatedOpenBISService service, + ISimpleLogger logger); +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/NonBalancer.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/NonBalancer.java new file mode 100644 index 00000000000..f983dfad13d --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/NonBalancer.java @@ -0,0 +1,65 @@ +/* + * Copyright 2011 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.etlserver.plugins; + +import static ch.systemsx.cisd.common.logging.LogLevel.INFO; + +import java.util.List; + +import org.apache.commons.io.FileUtils; + +import ch.systemsx.cisd.common.filesystem.FileUtilities; +import ch.systemsx.cisd.common.logging.ISimpleLogger; +import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; +import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share; +import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO; + +/** + * Implementation of {@link ISegmentedStoreBalancer} which just logs information about each share. + * + * @author Franz-Josef Elmer + */ +public class NonBalancer implements ISegmentedStoreBalancer +{ + + private static final int N = 3; + + public void balanceStore(List<Share> shares, IEncapsulatedOpenBISService service, + ISimpleLogger logger) + { + logger.log(INFO, "Data Store Shares:"); + for (Share share : shares) + { + List<SimpleDataSetInformationDTO> dataSets = share.getDataSetsOrderedBySize(); + logger.log(INFO, " Share " + share.getShareId() + " (free space: " + + FileUtils.byteCountToDisplaySize(share.calculateFreeSpace()) + ") has " + + dataSets.size() + " data sets occupying " + + FileUtilities.byteCountToDisplaySize(share.getTotalSizeOfDataSets()) + "."); + for (int i = 0, n = Math.min(N, dataSets.size()); i < n; i++) + { + SimpleDataSetInformationDTO dataSet = dataSets.get(i); + logger.log(INFO, " " + dataSet.getDataSetCode() + " " + + FileUtilities.byteCountToDisplaySize(dataSet.getDataSetSize())); + } + if (dataSets.size() > N) + { + logger.log(INFO, " ..."); + } + } + } + +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTask.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTask.java new file mode 100644 index 00000000000..4a1b8d8bf4f --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/plugins/SegmentedStoreBalancingTask.java @@ -0,0 +1,131 @@ +/* + * Copyright 2011 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.etlserver.plugins; + +import java.io.File; +import java.util.List; +import java.util.Properties; + +import org.apache.log4j.Logger; + +import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel; +import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; +import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider; +import ch.systemsx.cisd.common.filesystem.SimpleFreeSpaceProvider; +import ch.systemsx.cisd.common.logging.ISimpleLogger; +import ch.systemsx.cisd.common.logging.Log4jSimpleLogger; +import ch.systemsx.cisd.common.logging.LogCategory; +import ch.systemsx.cisd.common.logging.LogFactory; +import ch.systemsx.cisd.common.logging.LogInitializer; +import ch.systemsx.cisd.common.maintenance.IMaintenanceTask; +import ch.systemsx.cisd.common.utilities.ClassUtils; +import ch.systemsx.cisd.common.utilities.PropertyParametersUtil; +import ch.systemsx.cisd.common.utilities.PropertyUtils; +import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; +import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider; +import ch.systemsx.cisd.openbis.dss.generic.shared.utils.DssPropertyParametersUtil; +import ch.systemsx.cisd.openbis.dss.generic.shared.utils.SegmentedStoreUtils; +import ch.systemsx.cisd.openbis.dss.generic.shared.utils.Share; + +/** + * + * + * @author Franz-Josef Elmer + */ +public class SegmentedStoreBalancingTask implements IMaintenanceTask +{ + private static final String BALANCER_SECTION_NAME = "balancer"; + private static final String CLASS_PROPERTY_NAME = "class"; + + private static final Logger operationLog = + LogFactory.getLogger(LogCategory.OPERATION, SegmentedStoreBalancingTask.class); + + private final IEncapsulatedOpenBISService service; + private final IFreeSpaceProvider freeSpaceProvider; + private final ISimpleLogger operationLogger; + + private File storeRoot; + private String dataStoreCode; + private ISegmentedStoreBalancer balancer; + + public SegmentedStoreBalancingTask() + { + this(ServiceProvider.getOpenBISService(), new SimpleFreeSpaceProvider(), + new Log4jSimpleLogger(operationLog)); + } + + SegmentedStoreBalancingTask(IEncapsulatedOpenBISService service, + IFreeSpaceProvider freeSpaceProvider, ISimpleLogger logger) + { + LogInitializer.init(); + this.freeSpaceProvider = freeSpaceProvider; + this.service = service; + operationLogger = logger; + } + + public void setUp(String pluginName, Properties properties) + { + dataStoreCode = + PropertyUtils.getMandatoryProperty(properties, + DssPropertyParametersUtil.DSS_CODE_KEY); + storeRoot = + new File(PropertyUtils.getMandatoryProperty(properties, + DssPropertyParametersUtil.STOREROOT_DIR_KEY)); + if (storeRoot.isDirectory() == false) + { + throw new ConfigurationFailureException( + "Store root does not exists or is not a directory: " + + storeRoot.getAbsolutePath()); + } + balancer = createBalancer(properties); + operationLog.info("Plugin '" + pluginName + "' initialized: balancer: " + + balancer.getClass().getName() + ", data store code: " + dataStoreCode + + ", data store root: " + storeRoot.getAbsolutePath()); + } + + private ISegmentedStoreBalancer createBalancer(Properties properties) + { + Properties balancerProps = + PropertyParametersUtil.extractSingleSectionProperties(properties, + BALANCER_SECTION_NAME, false).getProperties(); + String className = balancerProps.getProperty(CLASS_PROPERTY_NAME); + if (className == null) + { + return new NonBalancer(); + } + try + { + return ClassUtils.create(ISegmentedStoreBalancer.class, className, balancerProps); + } catch (ConfigurationFailureException ex) + { + throw ex; + } catch (Exception ex) + { + throw new ConfigurationFailureException("Cannot find balancer class '" + className + + "'", CheckedExceptionTunnel.unwrapIfNecessary(ex)); + } + } + + public void execute() + { + List<Share> shares = + SegmentedStoreUtils.getDataSetsPerShare(storeRoot, dataStoreCode, + freeSpaceProvider, service, operationLogger); + balancer.balanceStore(shares, service, operationLogger); + } + +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DssPropertyParametersUtil.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DssPropertyParametersUtil.java index 6c69e1e09e0..ac09ed452b8 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DssPropertyParametersUtil.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/DssPropertyParametersUtil.java @@ -21,7 +21,6 @@ import java.util.Map.Entry; import java.util.Properties; import java.util.Set; -import ch.rinn.restrictions.Private; import ch.systemsx.cisd.common.filesystem.FileUtilities; import ch.systemsx.cisd.common.utilities.ExtendedProperties; import ch.systemsx.cisd.common.utilities.PropertyUtils; @@ -36,10 +35,8 @@ public class DssPropertyParametersUtil /** Prefix of system properties which may override service.properties. */ public static final String OPENBIS_DSS_SYSTEM_PROPERTIES_PREFIX = "openbis.dss."; - @Private - static final String DSS_CODE_KEY = "data-store-server-code"; + public static final String DSS_CODE_KEY = "data-store-server-code"; - @Private public static final String STOREROOT_DIR_KEY = "storeroot-dir"; public static final String DOWNLOAD_URL_KEY = "download-url"; diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java index d00f3bc11dd..6811824b748 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/SegmentedStoreUtils.java @@ -19,17 +19,30 @@ package ch.systemsx.cisd.openbis.dss.generic.shared.utils; import java.io.File; import java.io.FileFilter; import java.io.IOException; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Comparator; +import java.util.HashMap; +import java.util.List; +import java.util.Map; import java.util.regex.Pattern; +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.time.StopWatch; + import ch.systemsx.cisd.base.utilities.OSUtilities; import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; import ch.systemsx.cisd.common.exceptions.UserFailureException; import ch.systemsx.cisd.common.filesystem.FileUtilities; +import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider; import ch.systemsx.cisd.common.filesystem.rsync.RsyncCopier; +import ch.systemsx.cisd.common.logging.ISimpleLogger; +import ch.systemsx.cisd.common.logging.LogLevel; import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ExternalData; +import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO; /** * Utility methods for segmented stores. @@ -119,6 +132,67 @@ public class SegmentedStoreUtils "Now share could be found for the following incoming folder: " + testFile.getParentFile().getAbsolutePath()); } + + /** + * Gets a list of all shares of specified store root directory. As a side effect it calculates + * and updates the size of all data sets if necessary. + * + * @param dataStoreCode Code of the data store to which the root belongs. + * @param freeSpaceProvider Provider of free space used for all shares. + * @param service Access to openBIS API in order to get all data sets and to update data set + * size. + * @param log Logger for logging size calculations. + */ + public static List<Share> getDataSetsPerShare(File storeRoot, String dataStoreCode, + IFreeSpaceProvider freeSpaceProvider, IEncapsulatedOpenBISService service, + ISimpleLogger log) + { + Map<String, Share> shares = new HashMap<String, Share>(); + for (File file : getImcomingShares(storeRoot)) + { + Share share = new Share(file, freeSpaceProvider); + shares.put(share.getShareId(), share); + } + for (SimpleDataSetInformationDTO dataSet : service.listDataSets()) + { + String shareId = dataSet.getDataSetShareId(); + if (dataStoreCode.equals(dataSet.getDataStoreCode())) + { + Share share = shares.get(shareId); + File dataSetInStore = new File(share.getShare(), dataSet.getDataSetLocation()); + String dataSetCode = dataSet.getDataSetCode(); + if (dataSetInStore.exists()) + { + if (dataSet.getDataSetSize() == null) + { + StopWatch stopWatch = new StopWatch(); + log.log(LogLevel.INFO, "Calculating size of " + dataSetInStore); + stopWatch.start(); + long size = FileUtils.sizeOfDirectory(dataSetInStore); + stopWatch.stop(); + log.log(LogLevel.INFO, dataSetInStore + " contains " + size + + " bytes (calculated in " + stopWatch.getTime() + " msec)"); + service.updateShareIdAndSize(dataSetCode, shareId, size); + dataSet.setDataSetSize(size); + } + share.addDataSet(dataSet); + } else + { + log.log(LogLevel.WARN, "Data set " + dataSetCode + + " no longer exists in share " + shareId + "."); + } + } + } + List<Share> list = new ArrayList<Share>(shares.values()); + Collections.sort(list, new Comparator<Share>() + { + public int compare(Share o1, Share o2) + { + return o1.getShareId().compareTo(o2.getShareId()); + } + }); + return list; + } /** * Moves the specified data set to the specified share. The data set is folder in the store @@ -127,7 +201,8 @@ public class SegmentedStoreUtils * * @param service to access openBIS AS. */ - public static void moveDataSetToAnotherShare(File dataSetDirInStore, File share, IEncapsulatedOpenBISService service) + public static void moveDataSetToAnotherShare(File dataSetDirInStore, File share, + IEncapsulatedOpenBISService service) { String dataSetCode = dataSetDirInStore.getName(); ExternalData dataSet = service.tryGetDataSet(dataSetCode); @@ -135,7 +210,9 @@ public class SegmentedStoreUtils { throw new UserFailureException("Unknown data set " + dataSetCode); } - File oldShare = dataSetDirInStore.getParentFile().getParentFile().getParentFile().getParentFile().getParentFile(); + File oldShare = + dataSetDirInStore.getParentFile().getParentFile().getParentFile().getParentFile() + .getParentFile(); String relativePath = FileUtilities.getRelativeFile(oldShare, dataSetDirInStore); File dataSetDirInNewShare = new File(share, relativePath); dataSetDirInNewShare.mkdirs(); diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java new file mode 100644 index 00000000000..84c3783e226 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/utils/Share.java @@ -0,0 +1,118 @@ +/* + * Copyright 2011 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.dss.generic.shared.utils; + +import java.io.File; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Comparator; +import java.util.List; + +import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel; +import ch.systemsx.cisd.common.filesystem.HostAwareFile; +import ch.systemsx.cisd.common.filesystem.IFreeSpaceProvider; +import ch.systemsx.cisd.openbis.generic.shared.dto.SimpleDataSetInformationDTO; + +/** + * Represents a share of a segmented store. Holds the root directory of the share as well as + * the data sets. It is able to calculate the free disk space. + * + * @author Franz-Josef Elmer + */ +public final class Share +{ + private final File share; + + private final IFreeSpaceProvider freeSpaceProvider; + + private final String shareId; + + private final List<SimpleDataSetInformationDTO> dataSets = + new ArrayList<SimpleDataSetInformationDTO>(); + + private long size; + + Share(File share, IFreeSpaceProvider freeSpaceProvider) + { + this.share = share; + this.freeSpaceProvider = freeSpaceProvider; + shareId = share.getName(); + } + + /** + * Returns the share Id of this share. + */ + public String getShareId() + { + return shareId; + } + + /** + * Returns the root directory of this share. + */ + public File getShare() + { + return share; + } + + /** + * Calculates the actual free space (in bytes) of the hard disk on which this share resides. + */ + public long calculateFreeSpace() + { + try + { + return 1024 * freeSpaceProvider.freeSpaceKb(new HostAwareFile(share)); + } catch (IOException ex) + { + throw CheckedExceptionTunnel.wrapIfNecessary(ex); + } + } + + void addDataSet(SimpleDataSetInformationDTO dataSet) + { + dataSets.add(dataSet); + size += dataSet.getDataSetSize(); + } + + /** + * Returns all data sets of this shared ordered by size starting with the largest data set. + */ + public List<SimpleDataSetInformationDTO> getDataSetsOrderedBySize() + { + Collections.sort(dataSets, new Comparator<SimpleDataSetInformationDTO>() + { + public int compare(SimpleDataSetInformationDTO o1, + SimpleDataSetInformationDTO o2) + { + long size1 = o1.getDataSetSize(); + long size2 = o2.getDataSetSize(); + return size1 < size2 ? 1 : (size1 > size2 ? -1 : 0); + } + }); + return dataSets; + } + + /** + * Returns the total size (in bytes) of all data sets. + */ + public long getTotalSizeOfDataSets() + { + return size; + } +} \ No newline at end of file diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SimpleDataSetHelper.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SimpleDataSetHelper.java index 6e6cec89c0c..5a057cd6350 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SimpleDataSetHelper.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/server/business/bo/SimpleDataSetHelper.java @@ -50,6 +50,7 @@ public class SimpleDataSetHelper private static SimpleDataSetInformationDTO translate(ExternalDataPE data) { SimpleDataSetInformationDTO result = new SimpleDataSetInformationDTO(); + result.setDataStoreCode(data.getDataStore().getCode()); result.setDataSetCode(data.getCode()); result.setDataSetShareId(data.getShareId()); result.setDataSetLocation(data.getLocation()); diff --git a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/SimpleDataSetInformationDTO.java b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/SimpleDataSetInformationDTO.java index 06c924b3f12..52966542ec2 100644 --- a/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/SimpleDataSetInformationDTO.java +++ b/openbis/source/java/ch/systemsx/cisd/openbis/generic/shared/dto/SimpleDataSetInformationDTO.java @@ -29,6 +29,8 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.ServiceVersionHolder; public class SimpleDataSetInformationDTO implements Serializable { private static final long serialVersionUID = ServiceVersionHolder.VERSION; + + private String dataStoreCode; private String dataSetCode; @@ -52,6 +54,16 @@ public class SimpleDataSetInformationDTO implements Serializable private Collection<String> parentDataSetCodes; + public void setDataStoreCode(String dataStoreCode) + { + this.dataStoreCode = dataStoreCode; + } + + public String getDataStoreCode() + { + return dataStoreCode; + } + public String getDataSetType() { return dataSetType; -- GitLab