diff --git a/common/source/java/ch/systemsx/cisd/common/servlet/HttpServletRequestUtils.java b/common/source/java/ch/systemsx/cisd/common/servlet/HttpServletRequestUtils.java
index 9952b44a9667e7c86f23c360a0004908f2361ff4..01e075b8665d23b7d529d3d2b6e6dc5b0acba052 100644
--- a/common/source/java/ch/systemsx/cisd/common/servlet/HttpServletRequestUtils.java
+++ b/common/source/java/ch/systemsx/cisd/common/servlet/HttpServletRequestUtils.java
@@ -88,4 +88,29 @@ public class HttpServletRequestUtils
}
}
}
+
+ /**
+ * Returns a value of the specified request parameter as Boolean. If the value is null or the trimmed value is empty then returns False.
+ *
+ * @throws IllegalArgumentException when the parameter value is not a valid boolean
+ */
+ public static final Boolean getBooleanParameter(HttpServletRequest request, String parameterName)
+ {
+ String parameterValue = getStringParameter(request, parameterName);
+
+ if (parameterValue == null)
+ {
+ return Boolean.FALSE;
+ } else
+ {
+ try
+ {
+ return Boolean.valueOf(parameterValue);
+ } catch (NumberFormatException e)
+ {
+ throw new IllegalArgumentException("Parameter: " + parameterName
+ + " is not a valid boolean: " + parameterValue);
+ }
+ }
+ }
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/SessionWorkspaceFileUploadServlet.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/SessionWorkspaceFileUploadServlet.java
index 487fc3f94bee1646df2c6e651aae10763447d5d8..0f16735ec5edcc4998eb59e0d755db339480562f 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/SessionWorkspaceFileUploadServlet.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/SessionWorkspaceFileUploadServlet.java
@@ -16,6 +16,7 @@
package ch.systemsx.cisd.openbis.dss.generic.server;
+import java.io.File;
import java.io.IOException;
import java.io.InputStream;
import java.util.HashMap;
@@ -63,10 +64,7 @@ public class SessionWorkspaceFileUploadServlet extends HttpServlet
private static final String STATUS_PARAM = "status";
- /** If present, the file being uploaded is just an empty folder, not a file upload. */
- private static final String IS_EMPTY_FOLDER = "emptyFolder";
-
- private static final String UPLOAD_ID_PARAM = "uploadID";
+ private static final String IS_EMPTY_FOLDER = "emptyFolder"; // If present, is just an empty folder, not a file upload
private IDssServiceRpcGeneric service;
@@ -89,22 +87,29 @@ public class SessionWorkspaceFileUploadServlet extends HttpServlet
uploadRequest.validate();
- final String uploadId = uploadRequest.getUploadId();
- final String filePath = (uploadId != null)
- ? uploadId + "/" + uploadRequest.getFileName()
- : uploadRequest.getFileName();
- long bytes =
- service.putFileSliceToSessionWorkspace(uploadRequest.getSessionId(),
- filePath, uploadRequest.getStartByte(),
- uploadRequest.getFile());
+ long bytes = -1;
+ boolean isOk = false;
+ if (uploadRequest.isEmptyFolder())
+ {
+ File folder = service.putDirToSessionWorkspace(uploadRequest.getSessionId(),
+ uploadRequest.getFileName(), true);
+ isOk = folder.exists();
+ } else
+ {
+ bytes = service.putFileSliceToSessionWorkspace(uploadRequest.getSessionId(),
+ uploadRequest.getFileName(), uploadRequest.getStartByte(),
+ uploadRequest.getFile());
+ isOk = bytes >= 0;
+ }
Map<String, Object> resultMap = new HashMap<String, Object>();
resultMap.put(ID_PARAM, uploadRequest.getId());
resultMap.put(START_BYTE_PARAM, uploadRequest.getStartByte());
resultMap.put(END_BYTE_PARAM, uploadRequest.getEndByte());
resultMap.put(FILE_NAME_PARAM, uploadRequest.getFileName());
+ resultMap.put(IS_EMPTY_FOLDER, uploadRequest.isEmptyFolder());
resultMap.put(SIZE_PARAM, bytes);
- resultMap.put(STATUS_PARAM, "ok");
+ resultMap.put(STATUS_PARAM, (isOk)?"ok":"error");
SessionWorkspaceFileUploadResponse uploadResponse =
new SessionWorkspaceFileUploadResponse(response);
@@ -112,10 +117,10 @@ public class SessionWorkspaceFileUploadServlet extends HttpServlet
uploadResponse.writeJson(resultMap);
}
- private static class SessionWorkspaceFileUploadRequest
+ private class SessionWorkspaceFileUploadRequest
{
- private final HttpServletRequest request;
+ private HttpServletRequest request;
public SessionWorkspaceFileUploadRequest(HttpServletRequest request)
{
@@ -149,9 +154,9 @@ public class SessionWorkspaceFileUploadServlet extends HttpServlet
return HttpServletRequestUtils.getStringParameter(request, FILE_NAME_PARAM);
}
- public String getUploadId()
+ public Boolean isEmptyFolder()
{
- return HttpServletRequestUtils.getStringParameter(request, UPLOAD_ID_PARAM);
+ return HttpServletRequestUtils.getBooleanParameter(request, IS_EMPTY_FOLDER);
}
public InputStream getFile() throws IOException
@@ -186,10 +191,10 @@ public class SessionWorkspaceFileUploadServlet extends HttpServlet
}
- private static class SessionWorkspaceFileUploadResponse
+ private class SessionWorkspaceFileUploadResponse
{
- private final HttpServletResponse response;
+ private HttpServletResponse response;
public SessionWorkspaceFileUploadResponse(HttpServletResponse response)
{
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/api/v1/DssServiceRpcGeneric.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/api/v1/DssServiceRpcGeneric.java
index 52a57a244c3fb737a30bc18e91e55fc3a09ba2e7..73751b6d1f6f84481187d3e6e4d1e595d45b44f9 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/api/v1/DssServiceRpcGeneric.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/api/v1/DssServiceRpcGeneric.java
@@ -307,8 +307,7 @@ public class DssServiceRpcGeneric extends AbstractDssServiceRpc<IDssServiceRpcGe
}
@Override
- public long putFileSliceToSessionWorkspace(String sessionToken, String filePath,
- long slicePosition, InputStream sliceInputStream) throws IOExceptionUnchecked
+ public File putDirToSessionWorkspace(String sessionToken, String filePath, boolean isEmptyDirectory) throws IOExceptionUnchecked
{
getOpenBISService().checkSession(sessionToken);
if (filePath.contains("../"))
@@ -323,7 +322,17 @@ public class DssServiceRpcGeneric extends AbstractDssServiceRpc<IDssServiceRpcGe
final File dir = new File(workspaceDir, subDir);
dir.mkdirs();
final File file = new File(dir, filename);
+ if (isEmptyDirectory) {
+ file.mkdirs();
+ }
+ return file;
+ }
+ @Override
+ public long putFileSliceToSessionWorkspace(String sessionToken, String filePath,
+ long slicePosition, InputStream sliceInputStream) throws IOExceptionUnchecked
+ {
+ final File file = putDirToSessionWorkspace(sessionToken, filePath, false);
return FileUtilities.writeToFile(file, slicePosition, sliceInputStream);
}
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/api/v1/DssServiceRpcGenericLogger.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/api/v1/DssServiceRpcGenericLogger.java
index a67231e4a7518516a4856d98cef0e09d9caa7011..26c10fa231d89c46e2cd24f994ef07e27a7c516a 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/api/v1/DssServiceRpcGenericLogger.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/server/api/v1/DssServiceRpcGenericLogger.java
@@ -133,6 +133,12 @@ public class DssServiceRpcGenericLogger extends AbstractServerLogger implements
return 0;
}
+ @Override
+ public File putDirToSessionWorkspace(String sessionToken, String filePath, boolean isEmptyDirectory) throws IOExceptionUnchecked {
+ logTracking(sessionToken, "putDirToSessionWorkspace", "FILE_PATH(%s) EMPTY_DIRECTORY(%s)", filePath, isEmptyDirectory);
+ return null;
+ }
+
@Override
public long putFileSliceToSessionWorkspace(String sessionToken, String filePath,
long slicePosition, InputStream sliceInputStream) throws IOExceptionUnchecked
diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/api/v1/IDssServiceRpcGeneric.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/api/v1/IDssServiceRpcGeneric.java
index d6b5c32180be95ab4750a3c71f850febd456a9fa..1a01d007ca28fa365c495fb18e4597a67b7463b4 100644
--- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/api/v1/IDssServiceRpcGeneric.java
+++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/generic/shared/api/v1/IDssServiceRpcGeneric.java
@@ -16,6 +16,7 @@
package ch.systemsx.cisd.openbis.dss.generic.shared.api.v1;
+import java.io.File;
import java.io.InputStream;
import java.util.List;
import java.util.Map;
@@ -198,6 +199,18 @@ public interface IDssServiceRpcGeneric extends IRpcService
public long putFileToSessionWorkspace(String sessionToken, String filePath,
InputStream inputStream) throws IOExceptionUnchecked;
+ /**
+ * Upload a file slice to the user's session workspace. If the file does not exist then it will created.
+ *
+ * @param sessionToken The session token.
+ * @param filePath The file path (including the sub-directory) to upload the slice to.
+ * @param isEmptyDirectory Creates the file as an empty directory if true,
+ * returns the File object without creating it if false.
+ * @return File object representing the given path
+ * @throws IOExceptionUnchecked Thrown if IOException occurs.
+ */
+ public File putDirToSessionWorkspace(String sessionToken, String filePath, boolean isEmptyDirectory) throws IOExceptionUnchecked;
+
/**
* Upload a file slice to the user's session workspace. If the file does not exist then it will created.
*
diff --git a/openbis_api/source/java/ch/ethz/sis/openbis/generic/OpenBISAPI.java b/openbis_api/source/java/ch/ethz/sis/openbis/generic/OpenBISAPI.java
index 6e9d50baf699a7b4a005a8bdba170fb8773d37b8..e9a7c8798c3f7b0dc1d0f65f24ae34965993d53d 100644
--- a/openbis_api/source/java/ch/ethz/sis/openbis/generic/OpenBISAPI.java
+++ b/openbis_api/source/java/ch/ethz/sis/openbis/generic/OpenBISAPI.java
@@ -2,8 +2,12 @@ package ch.ethz.sis.openbis.generic;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.EntityKind;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.entitytype.id.EntityTypePermId;
+import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier;
import ch.ethz.sis.openbis.generic.dssapi.v3.IDataStoreServerApi;
import ch.ethz.sis.openbis.generic.dssapi.v3.dto.dataset.create.UploadedDataSetCreation;
+import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.http.JettyHttpClientFactory;
import ch.systemsx.cisd.common.logging.LogCategory;
import ch.systemsx.cisd.common.logging.LogFactory;
@@ -16,13 +20,7 @@ import java.io.IOException;
import java.io.InputStream;
import java.nio.file.Files;
import java.nio.file.Path;
-import java.util.Arrays;
-import java.util.Collection;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Objects;
-import java.util.UUID;
+import java.util.*;
import java.util.concurrent.ExecutionException;
import java.util.concurrent.TimeoutException;
import java.util.stream.Collectors;
@@ -42,7 +40,7 @@ public class OpenBISAPI {
private static final int CHUNK_SIZE = 1048576; // 1 MiB
- private static final Collection<Integer> ACCEPTABLE_STATUSES = List.of(200, 502);
+ private static final Collection<Integer> ACCEPTABLE_STATUSES = List.of(200);
private final IApplicationServerApi asFacade;
@@ -90,155 +88,132 @@ public class OpenBISAPI {
asFacade.logout(sessionToken);
}
- private List<File> contentOf(final File item)
+ public DataSetPermId createUploadedDataSet(final UploadedDataSetCreation newDataSet)
{
- if (item.getName().startsWith("."))
- {
- return Collections.emptyList();
- } else if (item.isFile())
- {
- return Collections.singletonList(item);
- } else
- {
- return Arrays.stream(Objects.requireNonNull(item.listFiles()))
- .flatMap(file -> contentOf(file).stream())
- .collect(Collectors.toList());
- }
+ return dssFacade.createUploadedDataSet(sessionToken, newDataSet);
}
- @SuppressWarnings("resource")
+
+
private Iterable<byte[]> streamFile(final File file, final int chunkSize) throws FileNotFoundException
{
final InputStream inputStream = new FileInputStream(file);
- return () -> new Iterator<>()
- {
- public boolean hasMore = true;
-
- public boolean hasNext()
- {
- return hasMore;
- }
+ return new Iterable<byte[]>() {
+ @Override
+ public Iterator<byte[]> iterator() {
+ return new Iterator<>() {
+ public boolean hasMore = true;
- public byte[] next()
- {
- try
- {
- byte[] bytes = inputStream.readNBytes(chunkSize);
- if (bytes.length < chunkSize)
- {
- hasMore = false;
- inputStream.close();
+ public boolean hasNext() {
+ return hasMore;
}
- return bytes;
- } catch (final IOException e)
- {
- try
- {
- inputStream.close();
- } catch (final IOException ex)
- {
- throw new RuntimeException(ex);
+
+ public byte[] next() {
+ try {
+ byte[] bytes = inputStream.readNBytes(chunkSize);
+ if (bytes.length < chunkSize) {
+ hasMore = false;
+ inputStream.close();
+ }
+ return bytes;
+ } catch (final IOException e) {
+ try {
+ inputStream.close();
+ } catch (final IOException ex) {
+ throw new RuntimeException(ex);
+ }
+ throw new RuntimeException(e);
+ }
}
- throw new RuntimeException(e);
- }
+ };
}
};
+ }
+
+ private String uploadFileWorkspaceDSSEmptyDir(String pathToDir) {
+ final org.eclipse.jetty.client.HttpClient client = JettyHttpClientFactory.getHttpClient();
+ final Request httpRequest = client.newRequest(dssURL + "/session_workspace_file_upload")
+ .method(HttpMethod.POST);
+ httpRequest.param("sessionID", sessionToken);
+ httpRequest.param("id", "1");
+ httpRequest.param("filename", pathToDir);
+ httpRequest.param("startByte", Long.toString(0));
+ httpRequest.param("endByte", Long.toString(0));
+ httpRequest.param("size", Long.toString(0));
+ httpRequest.param("emptyFolder", Boolean.TRUE.toString());
+
+ try {
+ final ContentResponse response = httpRequest.send();
+ final int status = response.getStatus();
+ if (status != 200)
+ {
+ throw new IOException(response.getContentAsString());
+ }
+ } catch (final IOException | TimeoutException | InterruptedException | ExecutionException e)
+ {
+ throw new RuntimeException(e);
+ }
+ return pathToDir;
+ }
+ private String uploadFileWorkspaceDSSFile(String pathToFile, File file) {
+ try {
+ long start = 0;
+ for (byte[] chunk : streamFile(file, CHUNK_SIZE)) {
+ final long end = start + chunk.length;
+
+ final org.eclipse.jetty.client.HttpClient client = JettyHttpClientFactory.getHttpClient();
+ final Request httpRequest = client.newRequest(dssURL + "/session_workspace_file_upload")
+ .method(HttpMethod.POST);
+ httpRequest.param("sessionID", sessionToken);
+ httpRequest.param("id", "1");
+ httpRequest.param("filename", pathToFile);
+ httpRequest.param("startByte", Long.toString(start));
+ httpRequest.param("endByte", Long.toString(end));
+ httpRequest.param("size", Long.toString(file.length()));
+
+ final ContentResponse response = httpRequest.send();
+ final int status = response.getStatus();
+ if (status != 200) {
+ throw new IOException(response.getContentAsString());
+ }
+ }
+ } catch (final IOException | TimeoutException | InterruptedException | ExecutionException e) {
+ throw new RuntimeException(e);
+ }
+ return pathToFile;
}
/**
* Upload file or folder to the DSS SessionWorkspaceFileUploadServlet and return the ID to be used by createUploadedDataSet
* This method hides the complexities of uploading a folder with many files and does the uploads in chunks.
*/
- public String uploadFileWorkspaceDSS(final Path fileOrFolder)
+ public String uploadFileWorkspaceDSS(final File fileOrFolder, final String... parents)
{
- // final ServiceFinder serviceFinder = new ServiceFinder("openbis", "session_workspace_file_upload");
- // serviceFinder.computeServerUrl()
-
- Objects.requireNonNull(sessionToken);
+ System.out.println("fileOrFolder: " + fileOrFolder);
- final List<File> content = contentOf(fileOrFolder.toFile());
- if (content.isEmpty())
+ if (fileOrFolder.exists() == false)
{
- throw new RuntimeException("The directory " + fileOrFolder + " is empty");
+ throw new UserFailureException("Path doesn't exist: " + fileOrFolder);
}
-
- final long totalSize = content.stream().reduce(0L, (size, f) ->
+ String fileNameOrFolderName = "";
+ if (parents.length == 1)
{
- try
- {
- return size + Files.size(f.toPath());
- } catch (IOException e)
- {
- throw new RuntimeException(e);
- }
- }, Long::sum);
+ fileNameOrFolderName = parents[0] + "/";
+ }
+ fileNameOrFolderName += fileOrFolder.getName();
- final String uploadId = UUID.randomUUID().toString();
- final org.eclipse.jetty.client.HttpClient client = JettyHttpClientFactory.getHttpClient();
- int id = 1;
- try
+ if (fileOrFolder.isDirectory())
{
- for (final File file : content)
+ uploadFileWorkspaceDSSEmptyDir(fileNameOrFolderName);
+ for (File file:fileOrFolder.listFiles())
{
- final String fileName = file.getName();
- final long fileSize = file.length();
- final String prefix = "(" + id + "/" + content.size() + ") " + fileName + ": ";
- final long size = Files.size(file.toPath());
- final long totalChunks = (size / CHUNK_SIZE) + (size % CHUNK_SIZE == 0 ? 0 : 1);
-
- OPERATION_LOG.info(prefix + "Starting upload of " + size + " bytes");
-
- long start = 0;
- for (var chunk : streamFile(file, CHUNK_SIZE))
- {
- if (chunk.length == 0)
- {
- continue;
- }
- final long end = start + chunk.length;
-
- int status = 0;
- while (status != 200)
- {
- final ContentProvider contentProvider = new BytesContentProvider(chunk);
-
- final Request httpRequest = client.newRequest(dssURL + "/session_workspace_file_upload")
- .method(HttpMethod.POST);
- httpRequest.param("sessionID", sessionToken);
- httpRequest.param("id", Integer.toString(id++));
- httpRequest.param("filename", fileName);
- httpRequest.param("startByte", Long.toString(start));
- httpRequest.param("endByte", Long.toString(end));
- httpRequest.param("size", Long.toString(fileSize));
- httpRequest.param("uploadID", uploadId);
- httpRequest.content(contentProvider);
- final ContentResponse response = httpRequest.send();
-
- status = response.getStatus();
- OPERATION_LOG.info(prefix + "Chunk " + (start / CHUNK_SIZE + 1) + "/" + totalChunks
- + " uploaded with status " + status);
-
- if (!ACCEPTABLE_STATUSES.contains(status))
- {
- throw new IOException(response.getContentAsString());
- }
- }
- start += CHUNK_SIZE;
- }
- OPERATION_LOG.info(prefix + "Upload complete");
+ uploadFileWorkspaceDSS(file, fileNameOrFolderName);
}
- } catch (final IOException | TimeoutException | InterruptedException | ExecutionException e)
- {
- throw new RuntimeException(e);
+ } else {
+ uploadFileWorkspaceDSSFile(fileNameOrFolderName, fileOrFolder);
}
-
- return uploadId;
- }
-
- public DataSetPermId createUploadedDataSet(final UploadedDataSetCreation newDataSet)
- {
- return dssFacade.createUploadedDataSet(sessionToken, newDataSet);
+ return fileNameOrFolderName;
}
}