diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dssapi/v3/executor/CreateUploadedDataSetExecutor.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dssapi/v3/executor/CreateUploadedDataSetExecutor.java index ae944447b24b1800d8e6c30226871c88166192f5..20b81a77c8ae6f88d7be1e2d8e04f42af2091b56 100644 --- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dssapi/v3/executor/CreateUploadedDataSetExecutor.java +++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dssapi/v3/executor/CreateUploadedDataSetExecutor.java @@ -16,14 +16,12 @@ package ch.ethz.sis.openbis.generic.server.dssapi.v3.executor; -import java.io.File; import java.io.IOException; import java.util.ArrayList; import java.util.Arrays; import java.util.List; import java.util.Map; -import org.apache.log4j.Logger; import org.springframework.stereotype.Component; import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi; @@ -46,8 +44,6 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.dataset.create.UploadedDataSetC import ch.ethz.sis.openbis.generic.server.asapi.v3.executor.OperationContext; import ch.ethz.sis.openbis.generic.server.asapi.v3.utils.ExceptionUtils; import ch.systemsx.cisd.common.exceptions.UserFailureException; -import ch.systemsx.cisd.common.logging.LogCategory; -import ch.systemsx.cisd.common.logging.LogFactory; import ch.systemsx.cisd.etlserver.api.v1.PutDataSetService; import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider; @@ -64,10 +60,6 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.dto.DataSetInformation; public class CreateUploadedDataSetExecutor implements ICreateUploadedDataSetExecutor { - private static Logger operationLog = LogFactory.getLogger(LogCategory.OPERATION, CreateUploadedDataSetExecutor.class); - - private PutDataSetService putService; - @Override public DataSetPermId create(String sessionToken, UploadedDataSetCreation creation) { @@ -173,28 +165,6 @@ public class CreateUploadedDataSetExecutor implements ICreateUploadedDataSetExec } } - // upload id - - if (creation.getUploadId() != null) - { - if (creation.getUploadId().contains("/")) - { - throw new UserFailureException("Upload id must not contain '/'"); - } - - File temporaryIncomingRoot = getPutService().getTemporaryIncomingRoot(typeCode); - File temporaryIncomingDir = new File(temporaryIncomingRoot, creation.getUploadId()); - - if (false == temporaryIncomingDir.exists()) - { - throw new UserFailureException( - "The folder for the upload id " + creation.getUploadId() + " could not be found. Have you uploaded the file(s) first?"); - } - } else - { - throw new UserFailureException("Upload id cannot be null."); - } - NewDataSetDTO newDataSet = new NewDataSetDTO(typeCode, owner, null, new ArrayList<FileInfoDssDTO>()); newDataSet.setProperties(creation.getProperties()); newDataSet.setParentDataSetCodes(parentCodes); @@ -211,14 +181,8 @@ public class CreateUploadedDataSetExecutor implements ICreateUploadedDataSetExec return ServiceProvider.getOpenBISService(); } - private synchronized PutDataSetService getPutService() + private PutDataSetService getPutService() { - if (putService == null) - { - putService = new PutDataSetService(ServiceProvider.getOpenBISService(), operationLog); - putService.setStoreDirectory(ServiceProvider.getConfigProvider().getStoreRoot()); - } - - return putService; + return (PutDataSetService) ServiceProvider.getDataStoreService().getPutDataSetService(); } } diff --git a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dssapi/v3/upload/StoreShareFileUploadServlet.java b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dssapi/v3/upload/StoreShareFileUploadServlet.java index 3ae957a46b1861b8f056dfd6891298e50b58d5ec..242a2203b6a03f3325eb97d7234624d8090522db 100644 --- a/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dssapi/v3/upload/StoreShareFileUploadServlet.java +++ b/datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dssapi/v3/upload/StoreShareFileUploadServlet.java @@ -19,7 +19,6 @@ package ch.ethz.sis.openbis.generic.server.dssapi.v3.upload; import java.io.IOException; import java.io.InputStream; -import javax.servlet.ServletConfig; import javax.servlet.ServletException; import javax.servlet.http.HttpServlet; import javax.servlet.http.HttpServletRequest; @@ -65,16 +64,6 @@ public class StoreShareFileUploadServlet extends HttpServlet public static final String UPLOAD_ID_PARAM = "uploadID"; - private PutDataSetService putService; - - @Override - public final void init(final ServletConfig servletConfig) throws ServletException - { - super.init(servletConfig); - this.putService = new PutDataSetService(ServiceProvider.getOpenBISService(), operationLog); - putService.setStoreDirectory(ServiceProvider.getConfigProvider().getStoreRoot()); - } - @Override protected void doPost(HttpServletRequest request, HttpServletResponse response) throws ServletException, IOException @@ -92,6 +81,8 @@ public class StoreShareFileUploadServlet extends HttpServlet throw new UserFailureException("Please upload at least one file"); } + PutDataSetService putService = (PutDataSetService) ServiceProvider.getDataStoreService().getPutDataSetService(); + while (iterator.hasNext()) { FileItemStream file = null; diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/DataSetTypeToRegistratorMapper.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/DataSetTypeToRegistratorMapper.java index 70c28293ea6292d024d8fc99f02eb1d4744cfa0c..6843a06f304fbd9b3fa0f67f39d86367514d9723 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/DataSetTypeToRegistratorMapper.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/DataSetTypeToRegistratorMapper.java @@ -17,6 +17,8 @@ package ch.systemsx.cisd.etlserver.api.v1; import java.io.File; +import java.util.ArrayList; +import java.util.Collection; import java.util.HashMap; import java.util.Properties; @@ -100,8 +102,7 @@ class DataSetTypeToRegistratorMapper + dropboxName + " = " + dropboxName); return defaultHandler; - } - else + } else { return plugin; } @@ -117,6 +118,14 @@ class DataSetTypeToRegistratorMapper return (null == plugin) ? defaultHandler : plugin; } + public Collection<ITopLevelDataSetRegistrator> getRegistrators() + { + Collection<ITopLevelDataSetRegistrator> registrators = new ArrayList<ITopLevelDataSetRegistrator>(); + registrators.add(defaultHandler); + registrators.addAll(handlerMap.values()); + return registrators; + } + public void initializeStoreRootDirectory(File storeDirectory) { initializeStoreRootDirectory(storeDirectory, defaultHandler); diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetService.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetService.java index 37f04bad595061aba16aba55a237726dfc832d14..8bd0b279faae8c11c174584a31200a3f803cb62c 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetService.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/api/v1/PutDataSetService.java @@ -21,6 +21,8 @@ import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Collection; import java.util.Collections; import java.util.List; import java.util.Properties; @@ -36,6 +38,7 @@ import ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel; import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked; import ch.systemsx.cisd.common.exceptions.ConfigurationFailureException; import ch.systemsx.cisd.common.exceptions.UserFailureException; +import ch.systemsx.cisd.common.filesystem.QueueingPathRemoverService; import ch.systemsx.cisd.common.mail.IMailClient; import ch.systemsx.cisd.common.mail.MailClient; import ch.systemsx.cisd.etlserver.DataStrategyStore; @@ -48,6 +51,7 @@ import ch.systemsx.cisd.openbis.common.io.ByteArrayBasedContentNode; import ch.systemsx.cisd.openbis.common.io.ConcatenatedContentInputStream; import ch.systemsx.cisd.openbis.dss.generic.shared.Constants; import ch.systemsx.cisd.openbis.dss.generic.shared.IEncapsulatedOpenBISService; +import ch.systemsx.cisd.openbis.dss.generic.shared.IPutDataSetService; import ch.systemsx.cisd.openbis.dss.generic.shared.ServiceProvider; import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.FileInfoDssDTO; import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTO; @@ -63,7 +67,7 @@ import ch.systemsx.cisd.openbis.generic.shared.basic.dto.DatabaseInstance; * * @author Chandrasekhar Ramakrishnan */ -public class PutDataSetService +public class PutDataSetService implements IPutDataSetService { private static final String MULTIPLE_FILES_UPLOAD_DIR = "upload"; @@ -270,15 +274,48 @@ public class PutDataSetService doInitialization(); } - String dataSetTypeOrNull = newDataSet.tryDataSetType(); - ITopLevelDataSetRegistrator registrator = registratorMap.getRegistratorForType(dataSetTypeOrNull); + if (StringUtils.isBlank(sessionToken)) + { + throw new UserFailureException("Session token cannot be null or empty"); + } + if (sessionToken.contains("/")) + { + throw new UserFailureException("Session token must not contain '/'"); + } + if (newDataSet == null) + { + throw new UserFailureException("New data set cannot be null"); + } + if (StringUtils.isBlank(uploadId)) + { + throw new UserFailureException("Upload id cannot be null or empty"); + } + if (uploadId.contains("/")) + { + throw new UserFailureException("Upload id must not contain '/'"); + } + + ServiceProvider.getOpenBISService().checkSession(sessionToken); + + String dataSetType = newDataSet.tryDataSetType(); + ITopLevelDataSetRegistrator registrator = registratorMap.getRegistratorForType(dataSetType); - File uploadDir = new File(getTemporaryIncomingRoot(dataSetTypeOrNull), uploadId); - File multipleFilesUploadDir = new File(uploadDir, MULTIPLE_FILES_UPLOAD_DIR); - File[] uploadedFiles = multipleFilesUploadDir.listFiles(); + File sessionUploadDir = new File(getTemporaryIncomingRoot(dataSetType), sessionToken); + File uploadIdDir = new File(sessionUploadDir, uploadId); + File multipleFilesUploadDir = new File(uploadIdDir, MULTIPLE_FILES_UPLOAD_DIR); + + File[] uploadedFiles = null; File dataSet = null; - if (uploadedFiles != null && uploadedFiles.length == 1) + if (multipleFilesUploadDir.exists() && multipleFilesUploadDir.isDirectory()) + { + uploadedFiles = multipleFilesUploadDir.listFiles(); + } + + if (uploadedFiles == null || uploadedFiles.length == 0) + { + throw new UserFailureException("No uploaded files found for upload id '" + uploadId + "'"); + } else if (uploadedFiles.length == 1) { dataSet = uploadedFiles[0]; } else @@ -288,11 +325,11 @@ public class PutDataSetService if (registrator instanceof PutDataSetServerPluginHolder) { - return new PutDataSetExecutor(this, ((PutDataSetServerPluginHolder) registrator).getPlugin(), sessionToken, newDataSet, uploadDir, + return new PutDataSetExecutor(this, ((PutDataSetServerPluginHolder) registrator).getPlugin(), sessionToken, newDataSet, uploadIdDir, dataSet).executeWithoutWriting(); } else { - return new PutDataSetTopLevelDataSetHandler(this, registrator, sessionToken, newDataSet, uploadDir, dataSet).executeWithoutWriting(); + return new PutDataSetTopLevelDataSetHandler(this, registrator, sessionToken, newDataSet, uploadIdDir, dataSet).executeWithoutWriting(); } } @@ -313,6 +350,10 @@ public class PutDataSetService { throw new UserFailureException("Session token cannot be null or empty"); } + if (sessionToken.contains("/")) + { + throw new UserFailureException("Session token must not contain '/'"); + } if (StringUtils.isBlank(filePath)) { throw new UserFailureException("File path cannot be null or empty"); @@ -344,27 +385,23 @@ public class PutDataSetService ServiceProvider.getOpenBISService().checkSession(sessionToken); - File uploadDir = new File(getTemporaryIncomingRoot(dataSetType), uploadId); - if (false == uploadDir.exists()) - { - uploadDir.mkdir(); - } - - File uploadSubDir = new File(uploadDir, MULTIPLE_FILES_UPLOAD_DIR); - if (false == uploadSubDir.exists()) - { - uploadSubDir.mkdir(); - } + File sessionUploadDir = new File(getTemporaryIncomingRoot(dataSetType), sessionToken); + File uploadIdDir = new File(sessionUploadDir, uploadId); + File multipleFilesUploadDir = new File(uploadIdDir, MULTIPLE_FILES_UPLOAD_DIR); File filePathDir = null; if (StringUtils.isBlank(folderPathOrNull)) { - filePathDir = new File(uploadSubDir, FilenameUtils.getPath(filePath)); + filePathDir = new File(multipleFilesUploadDir, FilenameUtils.getPath(filePath)); } else { - filePathDir = new File(uploadSubDir, FilenameUtils.getPath(folderPathOrNull + "/" + filePath)); + filePathDir = new File(multipleFilesUploadDir, FilenameUtils.getPath(folderPathOrNull + "/" + filePath)); + } + + if (false == filePathDir.exists()) + { + filePathDir.mkdirs(); } - filePathDir.mkdirs(); file = new File(filePathDir, FilenameUtils.getName(filePath)); outputStream = new FileOutputStream(file); @@ -468,6 +505,22 @@ public class PutDataSetService return registratorMap.getRegistratorForType(dataSetTypeOrNull).getGlobalState(); } + private Collection<TopLevelDataSetRegistratorGlobalState> getThreadGlobalStates() + { + Collection<TopLevelDataSetRegistratorGlobalState> states = new ArrayList<TopLevelDataSetRegistratorGlobalState>(); + Collection<ITopLevelDataSetRegistrator> registrators = registratorMap.getRegistrators(); + + for (ITopLevelDataSetRegistrator registrator : registrators) + { + if (registrator != null && registrator.getGlobalState() != null) + { + states.add(registrator.getGlobalState()); + } + } + + return states; + } + Logger getOperationLog() { return operationLog; @@ -526,6 +579,16 @@ public class PutDataSetService TopLevelDataSetRegistratorGlobalState globalState = getThreadGlobalState(dataSetTypeCodeOrNull); + return getTemporaryIncomingRoot(globalState); + } + + private File getTemporaryIncomingRoot(TopLevelDataSetRegistratorGlobalState globalState) + { + if (false == isInitialized) + { + doInitialization(); + } + File storeRoot = globalState.getStoreRootDir(); if (false == StringUtils.isBlank(globalState.getShareId())) { @@ -543,6 +606,46 @@ public class PutDataSetService return storeRoot; } + public void cleanupSession(String sessionToken) + { + if (false == isInitialized) + { + doInitialization(); + } + + if (StringUtils.isBlank(sessionToken)) + { + throw new IllegalArgumentException("Session token cannot be null or empty"); + } + if (sessionToken.contains("/")) + { + throw new UserFailureException("Session token must not contain '/'"); + } + + Collection<TopLevelDataSetRegistratorGlobalState> states = getThreadGlobalStates(); + + for (TopLevelDataSetRegistratorGlobalState state : states) + { + File sessionUploadDir = null; + + try + { + sessionUploadDir = new File(getTemporaryIncomingRoot(state), sessionToken); + + if (sessionUploadDir.exists()) + { + operationLog.info("Cleaning up a user session upload folder '" + sessionUploadDir.getAbsolutePath() + "'"); + QueueingPathRemoverService.removeRecursively(sessionUploadDir); + } + } catch (Exception e) + { + operationLog.warn( + "Could not clean up a user session upload folder '" + sessionUploadDir.getAbsolutePath() + "' together with the user session", + e); + } + } + } + } /** diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java index 15bfac52dd8cbd05ae3c5886e77f365123ff52b3..13697bc6c9a8e113896dc0ccf56a9cbc6167105c 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/DataStoreService.java @@ -511,6 +511,8 @@ public class DataStoreService extends AbstractServiceWithLogger<IDataStoreServic { QueueingPathRemoverService.removeRecursively(sessionWorkspace); } + + getPutDataSetService().cleanupSession(userSessionToken); } @Override @@ -587,7 +589,8 @@ public class DataStoreService extends AbstractServiceWithLogger<IDataStoreServic return availableService; } - private PutDataSetService getPutDataSetService() + @Override + public PutDataSetService getPutDataSetService() { if (putService == null) { diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataStoreServiceInternal.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataStoreServiceInternal.java index 8f88f2a3d3cbeb4ef5a2808a519473e5c95c49ff..4b186e7cd728867a720f4b6139ff76b09f921d2e 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataStoreServiceInternal.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IDataStoreServiceInternal.java @@ -72,4 +72,9 @@ public interface IDataStoreServiceInternal extends IInitializable, IDataStoreSer public void scheduleTask(String taskKey, IProcessingPluginTask task, Map<String, String> parameterBindings, List<DatasetDescription> datasets, String userId, String userEmailOrNull, String userSessionToken); + + /** + * Returns the put data set service. + */ + IPutDataSetService getPutDataSetService(); } diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IPutDataSetService.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IPutDataSetService.java new file mode 100644 index 0000000000000000000000000000000000000000..9fcb7e099a60b3e7e19c29e33d45194d7c17f68c --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/IPutDataSetService.java @@ -0,0 +1,25 @@ +/* + * Copyright 2018 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.dss.generic.shared; + +/** + * @author pkupczyk + */ +public interface IPutDataSetService +{ + +} diff --git a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateUploadedDataSetsTest.java b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateUploadedDataSetsTest.java index b928d5fb06dc67ab5059f8ae9f84ca3e2b628ff6..07eb6409ecb6923c36732bff7041ca785d5a3ce4 100644 --- a/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateUploadedDataSetsTest.java +++ b/datastore_server/sourceTest/java/ch/ethz/sis/openbis/generic/dss/systemtest/api/v3/CreateUploadedDataSetsTest.java @@ -16,6 +16,8 @@ package ch.ethz.sis.openbis.generic.dss.systemtest.api.v3; +import java.io.File; +import java.io.FilenameFilter; import java.io.IOException; import java.io.InputStream; import java.util.ArrayList; @@ -160,7 +162,7 @@ public class CreateUploadedDataSetsTest extends AbstractFileTest assertResponseError(response); assertTrue(response.getContentAsString(), response.getContentAsString().contains("File path cannot be null or empty")); } - + @Test public void testUploadWithFolderPathWithFolderUp() throws Exception { @@ -172,6 +174,52 @@ public class CreateUploadedDataSetsTest extends AbstractFileTest assertTrue(response.getContentAsString(), response.getContentAsString().contains("Folder path must not contain '../'")); } + @Test + public void testUploadWithFilesCleanedAfterLogout() throws Exception + { + String sessionToken = as.login(TEST_USER, PASSWORD); + + String uploadId = UUID.randomUUID().toString(); + String dataSetType = "UNKNOWN"; + + FileToUpload file = new FileToUpload("file", "test.txt", "test content"); + + ContentResponse response = uploadFiles(sessionToken, uploadId, dataSetType, false, null, file); + assertResponseOK(response); + + FilenameFilter sessionUploadDirFilter = new FilenameFilter() + { + @Override + public boolean accept(File dir, String name) + { + return sessionToken.equals(name); + } + }; + + File rpcIncomingDir = new File(store, "1/rpc-incoming"); + + File[] listFiles = rpcIncomingDir.listFiles(sessionUploadDirFilter); + assertEquals(1, listFiles.length); + + as.logout(sessionToken); + + // clean up of the session and the upload folder is done asynchronously therefore we have to wait + + long timeoutMillis = System.currentTimeMillis() + 5000; + while (System.currentTimeMillis() < timeoutMillis) + { + listFiles = rpcIncomingDir.listFiles(sessionUploadDirFilter); + if (listFiles.length == 0) + { + return; + } else + { + Thread.sleep(100); + } + } + fail("Session upload folder hasn't been removed after the logout"); + } + @Test public void testCreateWithInvalidSession() throws Exception { @@ -529,9 +577,7 @@ public class CreateUploadedDataSetsTest extends AbstractFileTest fail(); } catch (UserFailureException e) { - assertTrue(e.getMessage(), - e.getMessage() - .contains("The folder for the upload id " + uploadId + " could not be found. Have you uploaded the file(s) first?")); + assertTrue(e.getMessage(), e.getMessage().contains("No uploaded files found for upload id '" + uploadId + "'")); } } @@ -814,7 +860,7 @@ public class CreateUploadedDataSetsTest extends AbstractFileTest } catch (Exception e) { String fullStackTrace = ExceptionUtils.getFullStackTrace(e); - assertTrue(fullStackTrace, fullStackTrace.contains("The folder for the upload id " + uploadId + " could not be found")); + assertTrue(fullStackTrace, fullStackTrace.contains("No uploaded files found for upload id '" + uploadId + "'")); } }