From 2570a103eeac919deba1336375b188ff6c0f1b73 Mon Sep 17 00:00:00 2001
From: vkovtun <viktor.kovtun@id.ethz.ch>
Date: Tue, 19 Mar 2024 17:54:51 +0100
Subject: [PATCH] BIS-753: Made file upload work for small files.
---
.../ethz/sis/afsclient/client/AfsClient.java | 17 ++++++++++----
.../src/js/api/server-data-store-facade.js | 9 ++++----
.../src/js/demo/server-data-store-client.js | 2 +-
.../ch/ethz/sis/afs/api/OperationsAPI.java | 2 +-
.../sis/afs/dto/operation/WriteOperation.java | 6 ++---
.../ethz/sis/afs/exception/AFSExceptions.java | 2 +-
.../afs/manager/TransactionConnection.java | 6 ++---
.../operation/CreateOperationExecutor.java | 6 -----
.../operation/WriteOperationExecutor.java | 8 +++----
.../java/ch/ethz/sis/shared/io/IOUtils.java | 12 +++++++---
.../sis/afsserver/worker/AbstractProxy.java | 4 ++--
.../afsserver/worker/proxy/AuditorProxy.java | 4 ++--
.../worker/proxy/AuthenticationProxy.java | 4 ++--
.../worker/proxy/AuthorizationProxy.java | 4 ++--
.../sis/afsserver/worker/proxy/LogProxy.java | 6 ++---
.../worker/proxy/ValidationProxy.java | 4 ++--
ui-admin/index.html | 1 -
.../data-browser/DataBrowserController.js | 22 ++++++++++---------
18 files changed, 65 insertions(+), 54 deletions(-)
diff --git a/api-data-store-server-java/src/main/java/ch/ethz/sis/afsclient/client/AfsClient.java b/api-data-store-server-java/src/main/java/ch/ethz/sis/afsclient/client/AfsClient.java
index 3347bb51b79..3cf06bf9715 100644
--- a/api-data-store-server-java/src/main/java/ch/ethz/sis/afsclient/client/AfsClient.java
+++ b/api-data-store-server-java/src/main/java/ch/ethz/sis/afsclient/client/AfsClient.java
@@ -42,8 +42,6 @@ public final class AfsClient implements PublicAPI, ClientAPI
private static final int DEFAULT_TIMEOUT_IN_MILLIS = 30000;
- private static final String MD5 = "MD5";
-
private final int maxReadSizeInBytes;
private final int timeout;
@@ -361,7 +359,18 @@ public final class AfsClient implements PublicAPI, ClientAPI
{
try
{
- return MessageDigest.getInstance(MD5).digest(data);
+ return MessageDigest.getInstance("MD5").digest(data);
+ } catch (Exception exception)
+ {
+ throw new RuntimeException(exception);
+ }
+ }
+
+ private static byte[] getSHA1(final byte[] data)
+ {
+ try
+ {
+ return MessageDigest.getInstance("SHA-1").digest(data);
} catch (Exception exception)
{
throw new RuntimeException(exception);
@@ -592,7 +601,7 @@ public final class AfsClient implements PublicAPI, ClientAPI
final byte[] data = result < fullBuffer.length ? Arrays.copyOf(fullBuffer, result) : fullBuffer;
try
{
- final Boolean writeSuccessful = write(owner, destination, this.offset, data, getMD5(data));
+ final Boolean writeSuccessful = write(owner, destination, this.offset, data, getSHA1(data));
if (!writeSuccessful)
{
hasError.set(true);
diff --git a/api-data-store-server-javascript/src/js/api/server-data-store-facade.js b/api-data-store-server-javascript/src/js/api/server-data-store-facade.js
index 2cb686c301b..943f0bcbc88 100644
--- a/api-data-store-server-javascript/src/js/api/server-data-store-facade.js
+++ b/api-data-store-server-javascript/src/js/api/server-data-store-facade.js
@@ -439,16 +439,17 @@ function hex2a(hexx) {
* @param {str} owner owner of the file
* @param {str} source path to file
* @param {int} offset offset from which to start writing
- * @param {str} dataBase64 data to write in base64 format
+ * @param {str} base64Data data to write in base64 format
+ * @param {str} base64Hash MD5 of the base64 data
*/
-DataStoreServer.prototype.write = function(owner, source, offset, dataBase64){
+DataStoreServer.prototype.write = function(owner, source, offset, base64Data, base64Hash){
const params = this.fillCommonParameters({
"method": "write",
"owner" : owner,
"source": source,
"offset": offset,
- "data": dataBase64,
- "md5Hash": btoa(hex2a(md5(atob(dataBase64)))),
+ "data": base64Data,
+ "md5Hash": base64Hash
});
return this._internal.sendHttpRequest(
diff --git a/api-data-store-server-javascript/src/js/demo/server-data-store-client.js b/api-data-store-server-javascript/src/js/demo/server-data-store-client.js
index 2fee6a53822..8528af8c83a 100644
--- a/api-data-store-server-javascript/src/js/demo/server-data-store-client.js
+++ b/api-data-store-server-javascript/src/js/demo/server-data-store-client.js
@@ -193,7 +193,7 @@ window.onload = function() {
datastoreServer.write(owner,
document.getElementById("fpath").value.trim(),
parseInt(document.getElementById("foffset").value.trim()),
- btoa(document.getElementById("write-text").value.trim()))
+ document.getElementById("write-text").value.trim())
.then(() => showEntries());
}
};
diff --git a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/api/OperationsAPI.java b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/api/OperationsAPI.java
index 7ac51069f51..25ae7e158f6 100644
--- a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/api/OperationsAPI.java
+++ b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/api/OperationsAPI.java
@@ -29,7 +29,7 @@ public interface OperationsAPI {
@NonNull
byte[] read(@NonNull String source, @NonNull long offset, @NonNull int limit) throws Exception;
- boolean write(@NonNull String source, @NonNull long offset, @NonNull byte[] data, @NonNull byte[] md5Hash) throws Exception;
+ boolean write(@NonNull String source, @NonNull long offset, @NonNull byte[] data, @NonNull byte[] hash) throws Exception;
boolean delete(@NonNull String source) throws Exception;
diff --git a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/dto/operation/WriteOperation.java b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/dto/operation/WriteOperation.java
index 8048ff351d3..24ac74e6c31 100644
--- a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/dto/operation/WriteOperation.java
+++ b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/dto/operation/WriteOperation.java
@@ -36,17 +36,17 @@ public class WriteOperation implements Operation {
private String tempSource;
private long offset;
private byte[] data;
- private byte[] md5Hash;
+ private byte[] hash;
private OperationName name;
- public WriteOperation(UUID owner, String source, String tempSource, long offset, byte[] data, byte[] md5Hash) {
+ public WriteOperation(UUID owner, String source, String tempSource, long offset, byte[] data, byte[] hash) {
this.owner = owner;
this.locks = List.of(new Lock<>(owner, source, LockType.Exclusive));
this.source = source;
this.tempSource = tempSource;
this.offset = offset;
this.data = data;
- this.md5Hash = md5Hash;
+ this.hash = hash;
this.name = OperationName.Write;
}
}
diff --git a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/exception/AFSExceptions.java b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/exception/AFSExceptions.java
index c6a89190e62..e209906e182 100644
--- a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/exception/AFSExceptions.java
+++ b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/exception/AFSExceptions.java
@@ -40,7 +40,7 @@ public enum AFSExceptions implements ExceptionTemplateHolder {
PathInStore( RuntimeException.class, List.of(ClientDeveloperCodingError),10015,"Path given to: %s - In store: %s"),
PathInStoreCantBeRelative( RuntimeException.class, List.of(ClientDeveloperCodingError),10016,"Path given to: %s - can't contain '/../': %s"),
PathNotStartWithRoot( RuntimeException.class, List.of(ClientDeveloperCodingError),10017,"Path given to: %s - don't starts with root '/' : %s"),
- MD5NotMatch( RuntimeException.class, List.of(ClientDeveloperCodingError),10018,"MD5 doesn't match on data given to: %s - for: %s"),
+ HashNotMatch( RuntimeException.class, List.of(ClientDeveloperCodingError),10018,"Hash does not match data given to: %s - for: %s"),
DeadlockDetected( RuntimeException.class, List.of(UserUsageError), 10019,"Deadlock detected, %s is already waiting for %s from %s"),
TransactionReuse( RuntimeException.class, List.of(CoreDeveloperCodingError), 10020,"Transaction with uuid: %s and state: %s was going to be reused");
diff --git a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/TransactionConnection.java b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/TransactionConnection.java
index a02572ce815..6c5d9c32015 100644
--- a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/TransactionConnection.java
+++ b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/TransactionConnection.java
@@ -284,10 +284,10 @@ public class TransactionConnection implements TransactionalFileSystem {
private final Set<String> written = new HashSet<>();
@Override
- public boolean write(String source, long offset, byte[] data, byte[] md5Hash) throws Exception {
+ public boolean write(String source, long offset, byte[] data, byte[] hash) throws Exception {
source = getSafePath(OperationName.Write, source);
String tempSource = OperationExecutor.getTempPath(transaction, source) + "." + UUID.randomUUID();
- WriteOperation operation = new WriteOperation(transaction.getUuid(), source, tempSource, offset, data, md5Hash);
+ WriteOperation operation = new WriteOperation(transaction.getUuid(), source, tempSource, offset, data, hash);
boolean prepared = prepare(operation, source, null);
if (prepared) {
written.add(source);
@@ -380,7 +380,7 @@ public class TransactionConnection implements TransactionalFileSystem {
if (prepared) {
if (Objects.requireNonNull(operationName) == OperationName.Write)
{
- transaction.getOperations().add(((WriteOperation) operation).toBuilder().data(null).md5Hash(null).build());
+ transaction.getOperations().add(((WriteOperation) operation).toBuilder().data(null).hash(null).build());
} else
{
transaction.getOperations().add(operation);
diff --git a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/operation/CreateOperationExecutor.java b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/operation/CreateOperationExecutor.java
index 1d690946d29..ec852f4be0d 100644
--- a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/operation/CreateOperationExecutor.java
+++ b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/operation/CreateOperationExecutor.java
@@ -15,17 +15,11 @@
*/
package ch.ethz.sis.afs.manager.operation;
-import static ch.ethz.sis.afs.exception.AFSExceptions.MD5NotMatch;
import static ch.ethz.sis.afs.exception.AFSExceptions.PathInStore;
-import static ch.ethz.sis.afs.exception.AFSExceptions.PathIsDirectory;
-import java.util.Arrays;
-
-import ch.ethz.sis.afs.api.dto.File;
import ch.ethz.sis.afs.dto.Transaction;
import ch.ethz.sis.afs.dto.operation.CreateOperation;
import ch.ethz.sis.afs.dto.operation.OperationName;
-import ch.ethz.sis.afs.dto.operation.WriteOperation;
import ch.ethz.sis.afs.exception.AFSExceptions;
import ch.ethz.sis.shared.io.IOUtils;
import lombok.NonNull;
diff --git a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/operation/WriteOperationExecutor.java b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/operation/WriteOperationExecutor.java
index 97ab162e88a..a653ca089cf 100644
--- a/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/operation/WriteOperationExecutor.java
+++ b/lib-transactional-file-system/src/main/java/ch/ethz/sis/afs/manager/operation/WriteOperationExecutor.java
@@ -15,7 +15,7 @@
*/
package ch.ethz.sis.afs.manager.operation;
-import static ch.ethz.sis.afs.exception.AFSExceptions.MD5NotMatch;
+import static ch.ethz.sis.afs.exception.AFSExceptions.HashNotMatch;
import static ch.ethz.sis.afs.exception.AFSExceptions.PathIsDirectory;
import java.util.Arrays;
@@ -63,9 +63,9 @@ public class WriteOperationExecutor implements OperationExecutor<WriteOperation>
}
// 1. Validate new data
- byte[] md5Hash = IOUtils.getMD5(operation.getData());
- if (!Arrays.equals(md5Hash, operation.getMd5Hash())) {
- AFSExceptions.throwInstance(MD5NotMatch, OperationName.Write.name(), operation.getSource());
+ byte[] hash = IOUtils.getSHA1(operation.getData());
+ if (!Arrays.equals(hash, operation.getHash())) {
+ AFSExceptions.throwInstance(HashNotMatch, OperationName.Write.name(), operation.getSource());
}
// 2. Create temporary file if it has not been created already
boolean tempSourceExists = IOUtils.exists(operation.getTempSource());
diff --git a/lib-transactional-file-system/src/main/java/ch/ethz/sis/shared/io/IOUtils.java b/lib-transactional-file-system/src/main/java/ch/ethz/sis/shared/io/IOUtils.java
index ee574c4219d..f594ac6d9a6 100644
--- a/lib-transactional-file-system/src/main/java/ch/ethz/sis/shared/io/IOUtils.java
+++ b/lib-transactional-file-system/src/main/java/ch/ethz/sis/shared/io/IOUtils.java
@@ -620,11 +620,17 @@ public class IOUtils {
return pathAStore.equals(pathBStore);
}
- private static final String MD5 = "MD5";
-
public static byte[] getMD5(byte[] data) {
try {
- return MessageDigest.getInstance(MD5).digest(data);
+ return MessageDigest.getInstance("MD5").digest(data);
+ } catch (Exception exception) {
+ throw new RuntimeException(exception);
+ }
+ }
+
+ public static byte[] getSHA1(byte[] data) {
+ try {
+ return MessageDigest.getInstance("SHA-1").digest(data);
} catch (Exception exception) {
throw new RuntimeException(exception);
}
diff --git a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/AbstractProxy.java b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/AbstractProxy.java
index 6e075316ec5..aa1c65c7c37 100644
--- a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/AbstractProxy.java
+++ b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/AbstractProxy.java
@@ -144,8 +144,8 @@ public abstract class AbstractProxy implements Worker<TransactionalFileSystem> {
}
@Override
- public Boolean write(@NonNull String sourceOwner, @NonNull String source, @NonNull Long offset, @NonNull byte[] data, @NonNull byte[] md5Hash) throws Exception {
- return nextProxy.write(sourceOwner, source, offset, data, md5Hash);
+ public Boolean write(@NonNull String sourceOwner, @NonNull String source, @NonNull Long offset, @NonNull byte[] data, @NonNull byte[] hash) throws Exception {
+ return nextProxy.write(sourceOwner, source, offset, data, hash);
}
@Override
diff --git a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuditorProxy.java b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuditorProxy.java
index a9c0fafb1f8..1cb9d071020 100644
--- a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuditorProxy.java
+++ b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuditorProxy.java
@@ -94,9 +94,9 @@ public class AuditorProxy extends AbstractProxy {
}
@Override
- public Boolean write(@NonNull String owner, @NonNull String source, @NonNull Long offset, @NonNull byte[] data, @NonNull byte[] md5Hash) throws Exception {
+ public Boolean write(@NonNull String owner, @NonNull String source, @NonNull Long offset, @NonNull byte[] data, @NonNull byte[] hash) throws Exception {
auditBefore();
- return auditAfter(nextProxy.write(owner, source, offset, data, md5Hash));
+ return auditAfter(nextProxy.write(owner, source, offset, data, hash));
}
@Override
diff --git a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuthenticationProxy.java b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuthenticationProxy.java
index af3ca09d9be..985b22cc6e1 100644
--- a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuthenticationProxy.java
+++ b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuthenticationProxy.java
@@ -116,9 +116,9 @@ public class AuthenticationProxy extends AbstractProxy {
}
@Override
- public Boolean write(@NonNull String owner, @NonNull String source, @NonNull Long offset, @NonNull byte[] data, @NonNull byte[] md5Hash) throws Exception {
+ public Boolean write(@NonNull String owner, @NonNull String source, @NonNull Long offset, @NonNull byte[] data, @NonNull byte[] hash) throws Exception {
validateSessionAvailable();
- return nextProxy.write(owner, source, offset, data, md5Hash);
+ return nextProxy.write(owner, source, offset, data, hash);
}
@Override
diff --git a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuthorizationProxy.java b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuthorizationProxy.java
index 7790fa79bac..a20473747e3 100644
--- a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuthorizationProxy.java
+++ b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/AuthorizationProxy.java
@@ -60,9 +60,9 @@ public class AuthorizationProxy extends AbstractProxy {
}
@Override
- public Boolean write(String owner, String source, Long offset, byte[] data, byte[] md5Hash) throws Exception {
+ public Boolean write(String owner, String source, Long offset, byte[] data, byte[] hash) throws Exception {
validateUserRights(owner, source, IOUtils.writePermissions, OperationName.Write);
- return nextProxy.write(owner, source, offset, data, md5Hash);
+ return nextProxy.write(owner, source, offset, data, hash);
}
@Override
diff --git a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/LogProxy.java b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/LogProxy.java
index d2c6eeac7a6..da76ff85b3f 100644
--- a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/LogProxy.java
+++ b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/LogProxy.java
@@ -79,9 +79,9 @@ public class LogProxy extends AbstractProxy {
}
@Override
- public Boolean write(@NonNull String owner, @NonNull String source, @NonNull Long offset, @NonNull byte[] data, @NonNull byte[] md5Hash) throws Exception {
- logger.traceAccess(null, owner, source, offset, data.length, md5Hash.length);
- return logger.traceExit(nextProxy.write(owner, source, offset, data, md5Hash));
+ public Boolean write(@NonNull String owner, @NonNull String source, @NonNull Long offset, @NonNull byte[] data, @NonNull byte[] hash) throws Exception {
+ logger.traceAccess(null, owner, source, offset, data.length, hash.length);
+ return logger.traceExit(nextProxy.write(owner, source, offset, data, hash));
}
@Override
diff --git a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/ValidationProxy.java b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/ValidationProxy.java
index 42007bf196a..25f378c23e0 100644
--- a/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/ValidationProxy.java
+++ b/server-data-store/src/main/java/ch/ethz/sis/afsserver/worker/proxy/ValidationProxy.java
@@ -45,8 +45,8 @@ public class ValidationProxy extends AbstractProxy {
}
@Override
- public Boolean write(String owner, String source, Long offset, byte[] data, byte[] md5Hash) throws Exception {
- return nextProxy.write(owner, source, offset, data, md5Hash);
+ public Boolean write(String owner, String source, Long offset, byte[] data, byte[] hash) throws Exception {
+ return nextProxy.write(owner, source, offset, data, hash);
}
@Override
diff --git a/ui-admin/index.html b/ui-admin/index.html
index 0ad80de41a1..db5bd694f55 100644
--- a/ui-admin/index.html
+++ b/ui-admin/index.html
@@ -40,7 +40,6 @@
onerror="loadError()"
></script>
<script src="/openbis/resources/api/v3/require.js"></script>
- <script src="/openbis/resources/api/data-store-server/api/md5.js"></script>
<script src="/openbis/resources/api/data-store-server/api/server-data-store-facade.js"></script>
</head>
<body>
diff --git a/ui-admin/src/js/components/database/data-browser/DataBrowserController.js b/ui-admin/src/js/components/database/data-browser/DataBrowserController.js
index 92d1d0e20e8..d893a3730b8 100644
--- a/ui-admin/src/js/components/database/data-browser/DataBrowserController.js
+++ b/ui-admin/src/js/components/database/data-browser/DataBrowserController.js
@@ -169,27 +169,29 @@ export default class DataBrowserController extends ComponentController {
while (offset < file.size) {
const chunkData = await file.slice(offset, offset + CHUNK_SIZE).arrayBuffer()
// console.log(`Uploading chunk: ${offset} - Size: ${chunkData.byteLength}`)
- await this._uploadChunk(file.name, offset, await this._arrayBufferToBase64(chunkData))
+ await this._uploadChunk(file.name, offset, chunkData)
offset += CHUNK_SIZE
}
}
async _uploadChunk(source, offset, data) {
- // console.log(data)
- return await this.component.datastoreServer.write(this.owner, source, offset, data)
+ const hash = await crypto.subtle.digest("SHA-1", data)
+ const base64Data = await this._arrayBufferToBase64(data)
+ const base64Hash = await this._arrayBufferToBase64(hash)
+
+ return await this.component.datastoreServer.write(this.owner, source, offset, base64Data, base64Hash)
}
async _arrayBufferToBase64(buffer) {
return new Promise((resolve, reject) => {
- const blob = new Blob([buffer], {type: 'application/octet-stream'})
- const reader = new FileReader()
+ const blob = new Blob([buffer]);
+ const reader = new FileReader();
reader.onloadend = () => {
- const dataUrl = reader.result
- const base64String = dataUrl.split(',')[1]
- resolve(base64String)
+ const base64data = reader.result.split(',')[1];
+ resolve(base64data);
};
- reader.onerror = reject
- reader.readAsDataURL(blob)
+ reader.onerror = reject;
+ reader.readAsDataURL(blob);
});
}
--
GitLab