diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java new file mode 100644 index 0000000000000000000000000000000000000000..06a5d836fb670e87180b4b07f2e4147e546b14ad --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncArchiver.java @@ -0,0 +1,141 @@ +/* + * Copyright 2010 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard; + +import java.io.File; +import java.util.List; +import java.util.Properties; + +import ch.rinn.restrictions.Private; +import ch.systemsx.cisd.common.exceptions.Status; +import ch.systemsx.cisd.common.exceptions.UserFailureException; +import ch.systemsx.cisd.openbis.dss.generic.server.plugins.tasks.ArchiverTaskContext; +import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription; + +/** + * Archiver plugin which copies data sets to a destination folder using rsync (if it is remote). + * The destination can be + * <ul> + * <li>on the local file system, + * <li>a mounted remote folder, + * <li>a remote folder accessible via SSH, + * <li>a remote folder accessible via an rsync server. + * </ul> + * + * @author Piotr Buczek + */ +public class RsyncArchiver extends AbstractArchiverProcessingPlugin +{ + private static final long serialVersionUID = 1L; + + private transient RsyncDataSetCopier copier; + + private final IPathCopierFactory pathCopierFactory; + + private final ISshCommandExecutorFactory sshCommandExecutorFactory; + + public RsyncArchiver(Properties properties, File storeRoot) + { + this(properties, storeRoot, new RsyncCopierFactory(), new SshCommandExecutorFactory()); + } + + @Private + RsyncArchiver(Properties properties, File storeRoot, IPathCopierFactory pathCopierFactory, + ISshCommandExecutorFactory sshCommandExecutorFactory) + { + super(properties, storeRoot, null, null); + this.pathCopierFactory = pathCopierFactory; + this.sshCommandExecutorFactory = sshCommandExecutorFactory; + } + + private boolean isInitialized() + { + return copier != null; + } + + private void init() + { + this.copier = + new RsyncDataSetCopier(properties, pathCopierFactory, sshCommandExecutorFactory); + } + + private Status doArchive(DatasetDescription dataset, File originalData) + { + return copier.copyToDestination(originalData, dataset); + } + + private Status doUnarchive(DatasetDescription dataset, File originalData) + { + return copier.retrieveFromDestination(originalData, dataset); + } + + @Override + protected DatasetProcessingStatuses doArchive(List<DatasetDescription> datasets, + ArchiverTaskContext context) throws UserFailureException + { + if (isInitialized() == false) + { + init(); + } + DatasetProcessingStatuses statuses = new DatasetProcessingStatuses(); + for (DatasetDescription dataset : datasets) + { + File originalData = context.getDirectoryProvider().getDataSetDirectory(dataset); + Status status = doArchive(dataset, originalData); + statuses.addResult(dataset.getDatasetCode(), status, true); + } + + return statuses; + } + + @Override + protected DatasetProcessingStatuses doUnarchive(List<DatasetDescription> datasets, + ArchiverTaskContext context) throws UserFailureException + { + if (isInitialized() == false) + { + init(); + } + + // no need to lock - this is processing task + DatasetProcessingStatuses statuses = new DatasetProcessingStatuses(); + for (DatasetDescription dataset : datasets) + { + File originalData = context.getDirectoryProvider().getDataSetDirectory(dataset); + Status status = doUnarchive(dataset, originalData); + statuses.addResult(dataset.getDatasetCode(), status, false); + } + + return createStatusesFrom(Status.OK, datasets, false); + } + + protected boolean isProperlyArchived() throws UserFailureException + { + return true; + } + + protected void doDeleteFromArchive() throws UserFailureException + { + + } + + protected void doDeleteFromDss() throws UserFailureException + { + + } + +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopier.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopier.java new file mode 100644 index 0000000000000000000000000000000000000000..30d20a25562785d2d4bdad259bfe0d419ac706ab --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/plugins/standard/RsyncDataSetCopier.java @@ -0,0 +1,175 @@ +/* + * Copyright 2010 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.dss.generic.server.plugins.standard; + +import java.io.File; +import java.util.Properties; + +import org.apache.log4j.Logger; + +import ch.systemsx.cisd.common.exceptions.ExceptionWithStatus; +import ch.systemsx.cisd.common.exceptions.Status; +import ch.systemsx.cisd.common.filesystem.BooleanStatus; +import ch.systemsx.cisd.common.filesystem.FileOperations; +import ch.systemsx.cisd.common.filesystem.FileUtilities; +import ch.systemsx.cisd.common.filesystem.HostAwareFile; +import ch.systemsx.cisd.common.filesystem.IPathCopier; +import ch.systemsx.cisd.common.filesystem.ssh.ISshCommandExecutor; +import ch.systemsx.cisd.common.highwatermark.HostAwareFileWithHighwaterMark; +import ch.systemsx.cisd.common.logging.LogCategory; +import ch.systemsx.cisd.common.logging.LogFactory; +import ch.systemsx.cisd.common.utilities.PropertyUtils; +import ch.systemsx.cisd.openbis.dss.generic.server.IDataSetFileOperationsExecutor; +import ch.systemsx.cisd.openbis.dss.generic.server.LocalDataSetFileOperationsExcecutor; +import ch.systemsx.cisd.openbis.dss.generic.server.RemoteDataSetFileOperationsExecutor; +import ch.systemsx.cisd.openbis.generic.shared.dto.DatasetDescription; + +/** + * Code based on LocalAndRemoteCopier, able to copy dataset files both ways: to and from + * destination. + * + * @author Piotr Buczek + */ +public class RsyncDataSetCopier +{ + + private static final String DESTINATION_KEY = "destination"; + + private static final String RSYNC_PASSWORD_FILE_KEY = "rsync-password-file"; + + private static final String CHECK_EXISTENCE_FAILED = "couldn't check existence"; + + private static final String DESTINATION_DOES_NOT_EXIST = "destination doesn't exist"; + + private final static Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, + RsyncDataSetCopier.class); + + private static final String RSYNC_EXEC = "rsync"; + + private static final String SSH_EXEC = "ssh"; + + private static final long SSH_TIMEOUT_MILLIS = 15 * 1000; // 15s + + private final IDataSetFileOperationsExecutor executor; + + private final File destination; + + public RsyncDataSetCopier(Properties properties, IPathCopierFactory pathCopierFactory, + ISshCommandExecutorFactory sshCommandExecutorFactory) + { + String hostFile = PropertyUtils.getMandatoryProperty(properties, DESTINATION_KEY); + HostAwareFile hostAwareFile = HostAwareFileWithHighwaterMark.create(hostFile, -1); + String hostOrNull = hostAwareFile.tryGetHost(); + + this.destination = hostAwareFile.getFile(); + + if (hostOrNull == null) + { + this.executor = + new LocalDataSetFileOperationsExcecutor( + FileOperations.getMonitoredInstanceForCurrentThread()); + } else + { + File sshExecutable = Copier.getExecutable(properties, SSH_EXEC); + File rsyncExecutable = Copier.getExecutable(properties, RSYNC_EXEC); + IPathCopier copier = pathCopierFactory.create(rsyncExecutable, sshExecutable); + copier.check(); + String rsyncModule = hostAwareFile.tryGetRsyncModule(); + String rsyncPasswordFile = properties.getProperty(RSYNC_PASSWORD_FILE_KEY); + FileUtilities.checkPathCopier(copier, hostOrNull, null, rsyncModule, rsyncPasswordFile, + SSH_TIMEOUT_MILLIS); + ISshCommandExecutor sshCommandExecutor = + sshCommandExecutorFactory.create(sshExecutable, hostOrNull); + this.executor = + new RemoteDataSetFileOperationsExecutor(sshCommandExecutor, copier, hostOrNull, + rsyncModule, rsyncPasswordFile); + } + } + + /** + * Copies specified data file/folder to destination specified in constructor. The name of the + * file/folder at the destination is defined by the given data set code. + */ + public Status copyToDestination(File originalData, DatasetDescription dataset) + { + try + { + File destinationFolder = new File(destination, dataset.getDatasetCode()); + deleteDestinationFolder(destinationFolder); // cleanup needed for local executor + operationLog.info("Copy dataset '" + dataset.getDatasetCode() + "' from '" + + originalData.getPath() + "' to '" + destination.getPath()); + executor.copyDataSetToDestination(originalData, destination); + return Status.OK; + } catch (ExceptionWithStatus ex) + { + return ex.getStatus(); + } + } + + /** + * Retrieves specified data file/folder from the destination specified in constructor. The name + * of the file/folder at the destination is defined by the given data set code. + */ + public Status retrieveFromDestination(File originalData, DatasetDescription dataset) + { + try + { + File destinationFolder = new File(destination, dataset.getDatasetCode()); + checkDestinationExists(destinationFolder); + operationLog.info("Retrieve data set '" + dataset.getDatasetCode() + "' from '" + + destinationFolder.getPath() + "' to '" + + originalData.getParentFile().getPath()); + executor.retrieveDataSetFromDestination(originalData.getParentFile(), destinationFolder); + return Status.OK; + } catch (ExceptionWithStatus ex) + { + return ex.getStatus(); + } + } + + private void checkDestinationExists(File destinationFolder) + { + BooleanStatus destinationExists = destinationExists(destinationFolder); + if (destinationExists.isSuccess() == false) + { + operationLog.error("Destination folder '" + destinationFolder + "' doesn't exist"); + throw new ExceptionWithStatus(Status.createError(DESTINATION_DOES_NOT_EXIST)); + } + } + + private BooleanStatus destinationExists(File destinationFolder) + { + BooleanStatus destinationExists = executor.exists(destinationFolder); + if (destinationExists.isError()) + { + operationLog.error("Could not check existence of '" + destinationFolder + "': " + + destinationExists.tryGetMessage()); + throw new ExceptionWithStatus(Status.createError(CHECK_EXISTENCE_FAILED)); + } + return destinationExists; + } + + private void deleteDestinationFolder(File destinationFolder) + { + BooleanStatus destinationExists = destinationExists(destinationFolder); + if (destinationExists.isSuccess()) + { + executor.deleteFolder(destinationFolder); + } + } + +} \ No newline at end of file