Skip to content
Snippets Groups Projects
Commit e1804c1f authored by piotr.kupczyk@id.ethz.ch's avatar piotr.kupczyk@id.ethz.ch
Browse files

BIS-741 : AFS uses DSS store - make ExecutorProxy independent of V3 API

parent ac5c60c4
No related branches found
No related tags found
1 merge request!40SSDM-13578 : 2PT : Database and V3 Implementation - include the new AFS "free"...
Showing
with 214 additions and 195 deletions
......@@ -42,7 +42,8 @@ public enum AFSExceptions implements ExceptionTemplateHolder {
PathNotStartWithRoot( RuntimeException.class, List.of(ClientDeveloperCodingError),10017,"Path given to: %s - don't starts with root '/' : %s"),
MD5NotMatch( RuntimeException.class, List.of(ClientDeveloperCodingError),10018,"MD5 doesn't match on data given to: %s - for: %s"),
DeadlockDetected( RuntimeException.class, List.of(UserUsageError), 10019,"Deadlock detected, %s is already waiting for %s from %s"),
TransactionReuse( RuntimeException.class, List.of(CoreDeveloperCodingError), 10020,"Transaction with uuid: %s and state: %s was going to be reused");
TransactionReuse( RuntimeException.class, List.of(CoreDeveloperCodingError), 10020,"Transaction with uuid: %s and state: %s was going to be reused"),
NoSharesFound( RuntimeException.class, List.of(AdminConfigError), 10021,"No shares found");
private RuntimeExceptionTemplate template;
......
......@@ -35,7 +35,7 @@ public abstract class AbstractProxy implements Worker<TransactionalFileSystem> {
@Override
public void createContext(PerformanceAuditor performanceAuditor) {
setWorkerContext(new WorkerContext(performanceAuditor, null, null, null, null, false));
setWorkerContext(new WorkerContext(performanceAuditor, null, null, null, null, false, null, null, null));
}
@Override
......
......@@ -31,4 +31,7 @@ public class WorkerContext {
private String sessionToken;
private Boolean sessionExists;
private boolean transactionManagerMode;
private String ownerShareId;
private String[] ownerShards;
private String ownerFolder;
}
......@@ -15,6 +15,7 @@
*/
package ch.ethz.sis.afsserver.worker.providers;
import ch.ethz.sis.afsserver.worker.WorkerContext;
import ch.ethz.sis.shared.io.FilePermission;
import ch.ethz.sis.shared.startup.Configuration;
......@@ -23,5 +24,5 @@ import java.util.Set;
public interface AuthorizationInfoProvider {
void init(Configuration initParameter) throws Exception;
boolean doesSessionHaveRights(String sessionToken, String owner, Set<FilePermission> permissions);
boolean doesSessionHaveRights(WorkerContext workerContext, String owner, Set<FilePermission> permissions);
}
......@@ -15,6 +15,7 @@
*/
package ch.ethz.sis.afsserver.worker.providers.impl;
import ch.ethz.sis.afsserver.worker.WorkerContext;
import ch.ethz.sis.afsserver.worker.providers.AuthorizationInfoProvider;
import ch.ethz.sis.shared.io.FilePermission;
import ch.ethz.sis.shared.startup.Configuration;
......@@ -28,7 +29,7 @@ public class DummyAuthorizationInfoProvider implements AuthorizationInfoProvider
}
@Override
public boolean doesSessionHaveRights(String sessionToken, String owner, Set<FilePermission> permissions) {
public boolean doesSessionHaveRights(WorkerContext workerContext, String owner, Set<FilePermission> permissions) {
return true;
}
}
......@@ -17,17 +17,34 @@ package ch.ethz.sis.afsserver.worker.providers.impl;
import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Set;
import ch.ethz.sis.afsserver.startup.AtomicFileSystemServerParameter;
import ch.ethz.sis.afsserver.worker.WorkerContext;
import ch.ethz.sis.afsserver.worker.providers.AuthorizationInfoProvider;
import ch.ethz.sis.afsserver.worker.proxy.ProxyUtil;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.id.ObjectPermId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IPermIdHolder;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.IDataSetId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentPermId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.IExperimentId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.rights.Right;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.rights.Rights;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.rights.fetchoptions.RightsFetchOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.ISampleId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId;
import ch.ethz.sis.shared.io.FilePermission;
import ch.ethz.sis.shared.io.IOUtils;
import ch.ethz.sis.shared.startup.Configuration;
import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
......@@ -42,31 +59,94 @@ public class OpenBISAuthorizationInfoProvider implements AuthorizationInfoProvid
String openBISUrl = initParameter.getStringProperty(AtomicFileSystemServerParameter.openBISUrl);
int openBISTimeout = initParameter.getIntegerProperty(AtomicFileSystemServerParameter.openBISTimeout);
v3 = HttpInvokerUtils.createServiceStub(IApplicationServerApi.class, openBISUrl, openBISTimeout);
String storageUuid = initParameter.getStringProperty(AtomicFileSystemServerParameter.storageUuid);
if (storageUuid == null || storageUuid.isBlank())
{
throw new RuntimeException("Configuration parameter '" + AtomicFileSystemServerParameter.storageUuid + "' cannot be null or empty.");
}
}
@Override
public boolean doesSessionHaveRights(String sessionToken, String owner, Set<FilePermission> permissions)
public boolean doesSessionHaveRights(WorkerContext workerContext, String owner, Set<FilePermission> permissions)
{
IPermIdHolder foundOwner = ProxyUtil.findOwner(v3, sessionToken, owner);
String ownerShare = null;
ObjectPermId ownerPermId = null;
Set<FilePermission> ownerSupportedPermissions = null;
Experiment foundExperiment = findExperiment(workerContext.getSessionToken(), owner);
if (foundExperiment != null)
{
ownerPermId = foundExperiment.getPermId();
ownerSupportedPermissions = Set.of(FilePermission.Read, FilePermission.Write);
} else
{
Sample foundSample = findSample(workerContext.getSessionToken(), owner);
if (foundSample != null)
{
ownerPermId = foundSample.getPermId();
ownerSupportedPermissions = Set.of(FilePermission.Read, FilePermission.Write);
} else
{
DataSet foundDataSet = findDataSet(workerContext.getSessionToken(), owner);
if (foundOwner == null)
if (foundDataSet != null)
{
ownerPermId = foundDataSet.getPermId();
ownerShare = foundDataSet.getPhysicalData().getShareId();
ownerSupportedPermissions = Set.of(FilePermission.Read);
}
}
}
if (ownerPermId == null)
{
return false;
}
if (hasPermissions(workerContext, ownerPermId, ownerSupportedPermissions, permissions))
{
workerContext.setOwnerShareId(ownerShare);
workerContext.setOwnerShards(IOUtils.getShards(ownerPermId.getPermId()));
workerContext.setOwnerFolder(ownerPermId.getPermId());
return true;
} else
{
return false;
}
}
private boolean hasPermissions(WorkerContext workerContext, ObjectPermId ownerPermId, Set<FilePermission> ownerSupportedPermissions,
Set<FilePermission> requestedPermissions)
{
for (FilePermission requestPermission : requestedPermissions)
{
if (!ownerSupportedPermissions.contains(requestPermission))
{
return false;
}
}
Set<FilePermission> foundPermissions = new HashSet<>();
foundPermissions.add(FilePermission.Read);
Rights rights = v3.getRights(sessionToken, List.of(foundOwner.getPermId()), new RightsFetchOptions()).get(foundOwner.getPermId());
if (requestedPermissions.equals(foundPermissions))
{
return true;
}
Rights rights = v3.getRights(workerContext.getSessionToken(), List.of(ownerPermId), new RightsFetchOptions()).get(ownerPermId);
if (rights.getRights().contains(Right.UPDATE))
{
foundPermissions.add(FilePermission.Write);
}
for (FilePermission permission : permissions)
for (FilePermission requestedPermission : requestedPermissions)
{
if (!foundPermissions.contains(permission))
if (!foundPermissions.contains(requestedPermission))
{
return false;
}
......@@ -75,4 +155,87 @@ public class OpenBISAuthorizationInfoProvider implements AuthorizationInfoProvid
return true;
}
public IPermIdHolder findOwner(String sessionToken, String owner)
{
Experiment foundExperiment = findExperiment(sessionToken, owner);
if (foundExperiment != null)
{
return foundExperiment;
}
Sample foundSample = findSample(sessionToken, owner);
if (foundSample != null)
{
return foundSample;
}
return findDataSet(sessionToken, owner);
}
public Experiment findExperiment(String sessionToken, String experimentPermIdOrIdentifier)
{
IExperimentId experimentId;
if (experimentPermIdOrIdentifier.contains("/"))
{ // Is Identifier
experimentId = new ExperimentIdentifier(experimentPermIdOrIdentifier);
} else
{ // Is permId
experimentId = new ExperimentPermId(experimentPermIdOrIdentifier);
}
Map<IExperimentId, Experiment> experiments = v3.getExperiments(sessionToken, List.of(experimentId), new ExperimentFetchOptions());
if (!experiments.isEmpty())
{
return experiments.values().iterator().next();
} else
{
return null;
}
}
public Sample findSample(String sessionToken, String samplePermIdOrIdentifier)
{
ISampleId sampleId;
if (samplePermIdOrIdentifier.contains("/"))
{ // Is Identifier
sampleId = new SampleIdentifier(samplePermIdOrIdentifier);
} else
{ // Is permId
sampleId = new SamplePermId(samplePermIdOrIdentifier);
}
Map<ISampleId, Sample> samples = v3.getSamples(sessionToken, List.of(sampleId), new SampleFetchOptions());
if (!samples.isEmpty())
{
return samples.values().iterator().next();
} else
{
return null;
}
}
public DataSet findDataSet(String sessionToken, String dataSetPermId)
{
IDataSetId dataSetId = new DataSetPermId(dataSetPermId);
DataSetFetchOptions fo = new DataSetFetchOptions();
fo.withPhysicalData();
Map<IDataSetId, DataSet> dataSets = v3.getDataSets(sessionToken, List.of(dataSetId), fo);
if (!dataSets.isEmpty())
{
return dataSets.values().iterator().next();
} else
{
return null;
}
}
}
......@@ -39,7 +39,7 @@ public class AuthorizationProxy extends AbstractProxy {
}
private void validateUserRights(String owner, String source, Set<FilePermission> permissions, OperationName operationName) throws Exception {
boolean doesSessionHaveRights = authorizationInfoProvider.doesSessionHaveRights(workerContext.getSessionToken(),
boolean doesSessionHaveRights = authorizationInfoProvider.doesSessionHaveRights(workerContext,
owner,
permissions);
if (!doesSessionHaveRights) {
......
......@@ -18,6 +18,8 @@ package ch.ethz.sis.afsserver.worker.proxy;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Paths;
import java.util.Arrays;
import java.util.LinkedList;
import java.util.List;
import java.util.UUID;
import java.util.stream.Collectors;
......@@ -27,19 +29,13 @@ import ch.ethz.sis.afsapi.dto.File;
import ch.ethz.sis.afsapi.dto.FreeSpace;
import ch.ethz.sis.afsserver.startup.AtomicFileSystemServerParameter;
import ch.ethz.sis.afsserver.worker.AbstractProxy;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IPermIdHolder;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
import ch.ethz.sis.shared.io.IOUtils;
import ch.ethz.sis.shared.startup.Configuration;
import ch.systemsx.cisd.common.spring.HttpInvokerUtils;
import lombok.NonNull;
public class ExecutorProxy extends AbstractProxy
{
private final IApplicationServerApi v3;
private final String storageRoot;
private final String storageUuid;
......@@ -47,36 +43,8 @@ public class ExecutorProxy extends AbstractProxy
public ExecutorProxy(final Configuration configuration)
{
super(null);
storageRoot = configuration.getStringProperty(AtomicFileSystemServerParameter.storageRoot);
storageUuid = configuration.getStringProperty(AtomicFileSystemServerParameter.storageUuid);
if (storageUuid != null && !storageUuid.isBlank())
{
String openBISUrl = configuration.getStringProperty(AtomicFileSystemServerParameter.openBISUrl);
if (openBISUrl == null || openBISUrl.isBlank())
{
throw new RuntimeException(
"Incorrect configuration. '" + AtomicFileSystemServerParameter.openBISUrl + "' property is mandatory when '"
+ AtomicFileSystemServerParameter.storageUuid + "' is set.");
}
String openBISTimeout = configuration.getStringProperty(AtomicFileSystemServerParameter.openBISTimeout);
if (openBISTimeout == null || openBISTimeout.isBlank())
{
throw new RuntimeException(
"Incorrect configuration. '" + AtomicFileSystemServerParameter.openBISTimeout + "' property is mandatory when '"
+ AtomicFileSystemServerParameter.storageUuid + "' is set.");
}
v3 = HttpInvokerUtils.createServiceStub(IApplicationServerApi.class, openBISUrl,
configuration.getIntegerProperty(AtomicFileSystemServerParameter.openBISTimeout));
} else
{
v3 = null;
}
}
//
......@@ -118,57 +86,50 @@ public class ExecutorProxy extends AbstractProxy
// File System Operations
//
private String getOwnerPath(String shareId, String storageUuid, String[] shards, String ownerFolder)
{
List<String> elements = new LinkedList<>();
elements.add("");
elements.add(shareId);
elements.add(storageUuid);
elements.addAll(Arrays.asList(shards));
elements.add(ownerFolder);
return joinPaths(elements.toArray(new String[0]));
}
private String getOwnerPath(String owner)
{
if (storageUuid == null || storageUuid.isBlank())
if (workerContext.getOwnerShards() != null && workerContext.getOwnerFolder() != null)
{
// AFS does not reuse DSS store folder
return joinPaths("", owner);
} else
{
// AFS reuses DSS store folder
IPermIdHolder foundOwner = ProxyUtil.findOwner(v3, workerContext.getSessionToken(), owner);
if (foundOwner == null)
if (workerContext.getOwnerShareId() != null)
{
throw AFSExceptions.NotAPath.getInstance(owner);
}
String foundOwnerPath = null;
if (foundOwner instanceof DataSet)
{
DataSet foundDataSet = (DataSet) foundOwner;
foundOwnerPath = foundDataSet.getPhysicalData().getShareId() + "/" + foundDataSet.getPhysicalData().getLocation();
return getOwnerPath(workerContext.getOwnerShareId(), storageUuid, workerContext.getOwnerShards(), workerContext.getOwnerFolder());
} else
{
String[] shares = IOUtils.getShares(storageRoot);
if (shares.length == 0)
{
throw AFSExceptions.NotAPath.getInstance(owner);
}
String[] shards = IOUtils.getShards(owner);
for (String share : shares)
{
String potentialOwnerPath = share + "/" + storageUuid + "/" + String.join("/", shards) + "/" + foundOwner.getPermId().toString();
String potentialOwnerPath = getOwnerPath(share, storageUuid, workerContext.getOwnerShards(), workerContext.getOwnerFolder());
if (Files.exists(Paths.get(potentialOwnerPath)))
{
foundOwnerPath = potentialOwnerPath;
break;
return potentialOwnerPath;
}
}
if (foundOwnerPath == null)
if (shares.length > 0)
{
// if we don't find an existing owner folder at any share, then we will create it on the first share
foundOwnerPath = shares[0] + "/" + storageUuid + "/" + String.join("/", shards) + "/" + foundOwner.getPermId().toString();
return getOwnerPath(shares[0], storageUuid, workerContext.getOwnerShards(), workerContext.getOwnerFolder());
} else
{
throw AFSExceptions.NoSharesFound.getInstance();
}
}
return joinPaths("", foundOwnerPath);
} else
{
return joinPaths("", owner);
}
}
......@@ -185,19 +146,17 @@ public class ExecutorProxy extends AbstractProxy
@Override
public List<File> list(String owner, String source, Boolean recursively) throws Exception
{
String ownerPath = getOwnerPath(owner);
String sourcePath = joinPaths(ownerPath, source);
return workerContext.getConnection().list(sourcePath, recursively)
return workerContext.getConnection().list(getSourcePath(owner, source), recursively)
.stream()
.map(file -> convertToFile(owner, ownerPath, file))
.map(file -> convertToFile(owner, file))
.collect(Collectors.toList());
}
private File convertToFile(String owner, String ownerPath, ch.ethz.sis.afs.api.dto.File file)
private File convertToFile(String owner, ch.ethz.sis.afs.api.dto.File file)
{
try
{
String ownerFullPath = new java.io.File(joinPaths(this.storageRoot, ownerPath)).getCanonicalPath();
String ownerFullPath = new java.io.File(joinPaths(this.storageRoot, getOwnerPath(owner))).getCanonicalPath();
String fileFullPath;
if (file.getPath().startsWith(this.storageRoot))
......
package ch.ethz.sis.afsserver.worker.proxy;
import java.util.List;
import java.util.Map;
import ch.ethz.sis.openbis.generic.asapi.v3.IApplicationServerApi;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.common.interfaces.IPermIdHolder;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.DataSet;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.fetchoptions.DataSetFetchOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.DataSetPermId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.dataset.id.IDataSetId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.Experiment;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.fetchoptions.ExperimentFetchOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentIdentifier;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.ExperimentPermId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.experiment.id.IExperimentId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.Sample;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.fetchoptions.SampleFetchOptions;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.ISampleId;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SampleIdentifier;
import ch.ethz.sis.openbis.generic.asapi.v3.dto.sample.id.SamplePermId;
public class ProxyUtil
{
public static IPermIdHolder findOwner(IApplicationServerApi v3, String sessionToken, String owner)
{
Experiment foundExperiment = findExperiment(v3, sessionToken, owner);
if (foundExperiment != null)
{
return foundExperiment;
}
Sample foundSample = ProxyUtil.findSample(v3, sessionToken, owner);
if (foundSample != null)
{
return foundSample;
}
return ProxyUtil.findDataSet(v3, sessionToken, owner);
}
public static Experiment findExperiment(IApplicationServerApi v3, String sessionToken, String experimentPermIdOrIdentifier)
{
IExperimentId experimentId;
if (experimentPermIdOrIdentifier.contains("/"))
{ // Is Identifier
experimentId = new ExperimentIdentifier(experimentPermIdOrIdentifier);
} else
{ // Is permId
experimentId = new ExperimentPermId(experimentPermIdOrIdentifier);
}
Map<IExperimentId, Experiment> experiments = v3.getExperiments(sessionToken, List.of(experimentId), new ExperimentFetchOptions());
if (!experiments.isEmpty())
{
return experiments.values().iterator().next();
} else
{
return null;
}
}
public static Sample findSample(IApplicationServerApi v3, String sessionToken, String samplePermIdOrIdentifier)
{
ISampleId sampleId;
if (samplePermIdOrIdentifier.contains("/"))
{ // Is Identifier
sampleId = new SampleIdentifier(samplePermIdOrIdentifier);
} else
{ // Is permId
sampleId = new SamplePermId(samplePermIdOrIdentifier);
}
Map<ISampleId, Sample> samples = v3.getSamples(sessionToken, List.of(sampleId), new SampleFetchOptions());
if (!samples.isEmpty())
{
return samples.values().iterator().next();
} else
{
return null;
}
}
public static DataSet findDataSet(IApplicationServerApi v3, String sessionToken, String dataSetPermId)
{
IDataSetId dataSetId = new DataSetPermId(dataSetPermId);
DataSetFetchOptions fo = new DataSetFetchOptions();
fo.withPhysicalData();
Map<IDataSetId, DataSet> dataSets = v3.getDataSets(sessionToken, List.of(dataSetId), fo);
if (!dataSets.isEmpty())
{
return dataSets.values().iterator().next();
} else
{
return null;
}
}
}
......@@ -14,8 +14,8 @@ httpServerPort=8085
httpMaxContentLength=524288
maxReadSizeInBytes=524288
authenticationInfoProviderClass=ch.ethz.sis.afsserver.worker.providers.impl.DummyAuthenticationInfoProvider
authorizationInfoProviderClass=ch.ethz.sis.afsserver.worker.providers.impl.DummyAuthorizationInfoProvider
authenticationInfoProviderClass=ch.ethz.sis.afsserver.worker.providers.impl.OpenBISAuthenticationInfoProvider
authorizationInfoProviderClass=ch.ethz.sis.afsserver.worker.providers.impl.OpenBISAuthorizationInfoProvider
poolSize=50
connectionFactoryClass=ch.ethz.sis.afsserver.worker.ConnectionFactory
workerFactoryClass=ch.ethz.sis.afsserver.worker.WorkerFactory
......
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment