diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/ValidationError.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/validation/ValidationError.java similarity index 98% rename from datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/ValidationError.java rename to datastore_server/source/java/ch/systemsx/cisd/etlserver/validation/ValidationError.java index f2ee1863fc11da82fd397e631316ca3888ed9b7d..d5609f01593c6e4c9a17044b42f39cb0a5145711 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/ValidationError.java +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/validation/ValidationError.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package ch.systemsx.cisd.openbis.dss.client.api.gui; +package ch.systemsx.cisd.etlserver.validation; import org.apache.commons.lang.builder.ToStringBuilder; import org.apache.commons.lang.builder.ToStringStyle; diff --git a/datastore_server/source/java/ch/systemsx/cisd/etlserver/validation/ValidationScriptRunner.java b/datastore_server/source/java/ch/systemsx/cisd/etlserver/validation/ValidationScriptRunner.java new file mode 100644 index 0000000000000000000000000000000000000000..17d94dc45d7fbfa37e84062e4475fee1f8f7c0b8 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/etlserver/validation/ValidationScriptRunner.java @@ -0,0 +1,86 @@ +/* + * Copyright 2011 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.etlserver.validation; + +import java.io.File; +import java.util.ArrayList; +import java.util.Collection; +import java.util.List; + +import org.python.core.Py; +import org.python.core.PyFunction; +import org.python.core.PyObject; +import org.python.util.PythonInterpreter; + +import ch.systemsx.cisd.common.filesystem.FileUtilities; + +/** + * @author Chandrasekhar Ramakrishnan + */ +public class ValidationScriptRunner +{ + private final static String FILE_VALIDATION_FUNCTION_NAME = "validate_data_set_file"; + + private static String getValidationScriptString(File validationScriptFile) + { + String standardImports = "from " + ValidationError.class.getCanonicalName() + " import *"; + String scriptString = FileUtilities.loadToString(validationScriptFile); + return standardImports + "\n" + scriptString; + + // String scriptString = FileUtilities.loadToString(validationScriptFile); + // return scriptString; + } + + private final PythonInterpreter interpreter; + + private final String scriptPath; + + private final String scriptString; + + public ValidationScriptRunner(String scriptPath) + { + this.interpreter = new PythonInterpreter(); + this.scriptPath = scriptPath; + // Load the script + scriptString = getValidationScriptString(new File(this.scriptPath)); + + interpreter.exec(this.scriptString); + } + + @SuppressWarnings("unchecked") + public List<ValidationError> validate(File dataSetFile) + { + ArrayList<ValidationError> errors = new ArrayList<ValidationError>(); + PyFunction function = tryJythonFunction(FILE_VALIDATION_FUNCTION_NAME); + PyObject result = function.__call__(Py.java2py(dataSetFile)); + errors.addAll((Collection<? extends ValidationError>) result); + + return errors; + } + + private PyFunction tryJythonFunction(String functionName) + { + try + { + PyFunction function = (PyFunction) interpreter.get(functionName, PyFunction.class); + return function; + } catch (Exception e) + { + return null; + } + } +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/CommandPut.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/CommandPut.java index 37c2934d888ecd3025a8d904fc869f4eb1512822..d002fa2504cfecb895aa27f73a949ff7b7b1b041 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/CommandPut.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/CommandPut.java @@ -36,7 +36,7 @@ import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTO.DataSetO import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTO.DataSetOwnerType; /** - * Command that lists files in the data set. + * Command that uploads a data set. * * @author Chandrasekhar Ramakrishnan */ diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/CommandTestValid.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/CommandTestValid.java new file mode 100644 index 0000000000000000000000000000000000000000..679ce8f9f222c1ea63822df6c0eaedf1d831de99 --- /dev/null +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/CommandTestValid.java @@ -0,0 +1,333 @@ +/* + * Copyright 2010 ETH Zuerich, CISD + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package ch.systemsx.cisd.openbis.dss.client.api.cli; + +import java.io.File; +import java.io.IOException; +import java.io.PrintStream; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; + +import ch.systemsx.cisd.args4j.ExampleMode; +import ch.systemsx.cisd.args4j.Option; +import ch.systemsx.cisd.base.exceptions.IOExceptionUnchecked; +import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException; +import ch.systemsx.cisd.common.exceptions.InvalidSessionException; +import ch.systemsx.cisd.common.exceptions.UserFailureException; +import ch.systemsx.cisd.etlserver.validation.ValidationError; +import ch.systemsx.cisd.etlserver.validation.ValidationScriptRunner; +import ch.systemsx.cisd.openbis.dss.client.api.v1.IDataSetDss; +import ch.systemsx.cisd.openbis.dss.client.api.v1.IDssComponent; +import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.FileInfoDssBuilder; +import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.FileInfoDssDTO; +import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTO; +import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTO.DataSetOwner; +import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTO.DataSetOwnerType; + +/** + * Command that runs a validation script and returns the error messages if the data set is not + * valid. + * + * @author Chandrasekhar Ramakrishnan + */ +class CommandTestValid extends AbstractDssCommand<CommandTestValid.CommandTestValidArguments> +{ + static class CommandTestValidArguments extends GlobalArguments + { + @Option(name = "t", longName = "type", usage = "Set the data set type") + private String dataSetType; + + @Option(longName = "props", usage = "Set properties of the data set (format: code=val[,code=val]*") + private String propertiesString; + + public String getDataSetType() + { + return dataSetType; + } + + public DataSetOwnerType getOwnerType() + { + return DataSetOwnerType.valueOf(getArguments().get(0).toString().toUpperCase()); + } + + public String getOwnerIdentifier() + { + return getArguments().get(1); + } + + public String getFilePath() + { + return getArguments().get(2); + } + + public File getFile() + { + return new File(getFilePath()); + } + + public String getScriptPath() + { + return getArguments().get(3); + } + + public HashMap<String, String> getProperties() + { + HashMap<String, String> propsMap = new HashMap<String, String>(); + String propsString = propertiesString; + if (propsString == null || propsString.length() == 0) + { + return propsMap; + } + + String[] propsArray = propsString.split(","); + for (String propLine : propsArray) + { + String[] keyAndValue = propLine.split("="); + assert keyAndValue.length == 2; + propsMap.put(keyAndValue[0], keyAndValue[1]); + } + return propsMap; + } + + @Override + public boolean isComplete() + { + if (getArguments().size() < 4) + return false; + + try + { + getOwnerType(); + } catch (IllegalArgumentException e) + { + return false; + } + + try + { + getProperties(); + } catch (Exception e) + { + System.err + .println("\nProprties must be specified using as code=value[,code=value]*\n"); + return false; + } + + try + { + String path = getScriptPath(); + // Require a script at the moment. In the future, this might change to validate + // against the script on the server. + if (null == path) + { + return false; + } + + File scriptFile = new File(path); + if (false == scriptFile.exists()) + { + System.err.println("\n" + path + + " does not exist. Please specify a python (jython) script."); + return false; + } + + } catch (Exception e) + { + System.err.println("\nThe script must be a valid python (jython) script."); + } + + if (false == super.isComplete()) + return false; + + return true; + } + } + + private static class CommandTestValidExecutor extends + AbstractExecutor<CommandTestValidArguments> + { + + CommandTestValidExecutor(CommandTestValidArguments arguments, + AbstractDssCommand<CommandTestValidArguments> command) + { + super(arguments, command); + } + + @Override + protected ResultCode doExecute(IDssComponent component) + { + try + { + NewDataSetDTO newDataSet = getNewDataSet(); + if (newDataSet.getFileInfos().isEmpty()) + { + File file = arguments.getFile(); + if (false == file.exists()) + { + System.err.println("Data set file does not exist"); + } else if (false == file.isDirectory()) + { + System.err.println("Must select a directory to upload."); + } else + { + System.err.println("Data set is empty."); + } + return ResultCode.INVALID_ARGS; + } + ValidationScriptRunner scriptRunner = + new ValidationScriptRunner(arguments.getScriptPath()); + + List<ValidationError> errors = scriptRunner.validate(arguments.getFile()); + for (ValidationError error : errors) + { + System.out.println(error.getErrorMessage()); + } + } catch (IOException e) + { + throw new IOExceptionUnchecked(e); + } + + return ResultCode.OK; + } + + private NewDataSetDTO getNewDataSet() throws IOException + { + // Get the owner + // That the owner type is valid has already been checked by CmdPutArguments#isComplete + DataSetOwnerType ownerType = arguments.getOwnerType(); + String ownerIdentifier = arguments.getOwnerIdentifier(); + DataSetOwner owner = new NewDataSetDTO.DataSetOwner(ownerType, ownerIdentifier); + + File file = arguments.getFile(); + ArrayList<FileInfoDssDTO> fileInfos = getFileInfosForPath(file); + + // Get the parent + String parentNameOrNull = null; + if (file.isDirectory()) + { + parentNameOrNull = file.getName(); + } + + NewDataSetDTO dataSet = new NewDataSetDTO(owner, parentNameOrNull, fileInfos); + // Set the data set type (may be null) + dataSet.setDataSetTypeOrNull(arguments.getDataSetType()); + + // Set the properties + dataSet.setProperties(arguments.getProperties()); + + return dataSet; + } + + private ArrayList<FileInfoDssDTO> getFileInfosForPath(File file) throws IOException + { + ArrayList<FileInfoDssDTO> fileInfos = new ArrayList<FileInfoDssDTO>(); + if (false == file.exists()) + { + return fileInfos; + } + + String path = file.getCanonicalPath(); + if (false == file.isDirectory()) + { + path = file.getParentFile().getCanonicalPath(); + } + + FileInfoDssBuilder builder = new FileInfoDssBuilder(path, path); + builder.appendFileInfosForFile(file, fileInfos, true); + return fileInfos; + } + } + + CommandTestValid() + { + super(new CommandTestValidArguments()); + } + + public ResultCode execute(String[] args) throws UserFailureException, + EnvironmentFailureException + { + return new CommandTestValidExecutor(arguments, this).execute(args); + } + + public String getName() + { + return "testvalid"; + } + + /** + * Print usage information about the command. + */ + @Override + public void printUsage(PrintStream out) + { + out.println(getUsagePrefixString() + " [options] " + getRequiredArgumentsString()); + parser.printUsage(out); + out.println(" Examples : "); + out.println(" " + getCommandCallString() + parser.printExample(ExampleMode.ALL) + + " EXPERIMENT <experiment identifier> <path> <script>"); + out.println(" " + getCommandCallString() + parser.printExample(ExampleMode.ALL) + + " SAMPLE <sample identifier> <path> <script>"); + } + + @Override + protected String getRequiredArgumentsString() + { + return "<owner type> <owner> <path> <script>"; + } + + /** + * Creates the DSS Component object and logs into the server. + */ + @Override + protected IDssComponent login(GlobalArguments args) + { + // Create a dummy component. + IDssComponent component = new IDssComponent() + { + + public void checkSession() throws InvalidSessionException + { + + } + + public String getSessionToken() throws IllegalStateException + { + return null; + } + + public IDataSetDss getDataSet(String code) throws IllegalStateException, + EnvironmentFailureException + { + return null; + } + + public IDataSetDss putDataSet(NewDataSetDTO newDataset, File dataSetFile) + throws IllegalStateException, EnvironmentFailureException + { + return null; + } + + public void logout() + { + + } + + }; + return component; + } +} diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/DssCommandFactory.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/DssCommandFactory.java index 8d4ebc468988c8603f1297621b9c0d4702497dbd..d217b52d7bf3b6832adc29c3216797aa78f98177 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/DssCommandFactory.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/cli/DssCommandFactory.java @@ -28,7 +28,7 @@ public class DssCommandFactory extends AbstractCommandFactory private static enum Command { - LS, GET, HELP, PUT + LS, GET, HELP, PUT, TESTVALID } public ICommand tryCommandForName(String name) @@ -63,6 +63,9 @@ public class DssCommandFactory extends AbstractCommandFactory case PUT: result = new CommandPut(); break; + case TESTVALID: + result = new CommandTestValid(); + break; default: result = null; break; @@ -79,7 +82,7 @@ public class DssCommandFactory extends AbstractCommandFactory public List<String> getKnownCommands() { String[] commands = - { "ls", "get", "put" }; + { "ls", "get", "put", "testvalid" }; return Arrays.asList(commands); } } diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetMetadataPanel.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetMetadataPanel.java index 6bf838be637794609ef3417a5bf7e1ca2eec789a..a6c09e8481c77b36eadbad2e382279dcd00c1037 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetMetadataPanel.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetMetadataPanel.java @@ -51,6 +51,7 @@ import javax.swing.JPanel; import javax.swing.JRadioButton; import javax.swing.JTextField; +import ch.systemsx.cisd.etlserver.validation.ValidationError; import ch.systemsx.cisd.openbis.dss.client.api.gui.DataSetUploadClientModel.NewDataSetInfo; import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTO.DataSetOwnerType; import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTOBuilder; diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetPropertiesPanel.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetPropertiesPanel.java index 1810e68ab7f4191f1acda6d096e581bb0a4da48f..975ba430669a9d4c88188f24027f633e3a07e229 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetPropertiesPanel.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetPropertiesPanel.java @@ -45,6 +45,7 @@ import javax.swing.JPanel; import javax.swing.JTextField; import ch.systemsx.cisd.common.utilities.PropertyUtils; +import ch.systemsx.cisd.etlserver.validation.ValidationError; import ch.systemsx.cisd.openbis.dss.client.api.gui.DataSetUploadClientModel.NewDataSetInfo; import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetDTOBuilder; import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.NewDataSetMetadataDTO; diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetUploadClientModel.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetUploadClientModel.java index 88446664bd589b99a3080d3c070be919cfcd13a0..4cbcb76f546d6ed9228e827eff4b360687a920cb 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetUploadClientModel.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/DataSetUploadClientModel.java @@ -30,6 +30,7 @@ import ch.systemsx.cisd.base.namedthread.NamingThreadPoolExecutor; import ch.systemsx.cisd.common.exceptions.UserFailureException; import ch.systemsx.cisd.common.io.TransmissionSpeedCalculator; import ch.systemsx.cisd.common.utilities.ITimeProvider; +import ch.systemsx.cisd.etlserver.validation.ValidationError; import ch.systemsx.cisd.openbis.dss.client.api.gui.DataSetUploadClientModel.NewDataSetInfo.Status; import ch.systemsx.cisd.openbis.dss.client.api.v1.IDssComponent; import ch.systemsx.cisd.openbis.dss.generic.shared.api.v1.FileInfoDssDTO; diff --git a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/UiUtilities.java b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/UiUtilities.java index e146055ddf95b56976df7c40ae3432a96130039c..8d74a52e60981146e271bb0da7216a6a61ff4ae0 100644 --- a/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/UiUtilities.java +++ b/datastore_server/source/java/ch/systemsx/cisd/openbis/dss/client/api/gui/UiUtilities.java @@ -21,6 +21,8 @@ import java.awt.Color; import javax.swing.JComponent; import javax.swing.JLabel; +import ch.systemsx.cisd.etlserver.validation.ValidationError; + /** * @author Chandrasekhar Ramakrishnan */ diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/validation/basic-validation-script.py b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/validation/basic-validation-script.py new file mode 100644 index 0000000000000000000000000000000000000000..109e3d92e2f1cd670508992f9828e55b362fd541 --- /dev/null +++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/validation/basic-validation-script.py @@ -0,0 +1,13 @@ +import os + +def validate_data_set_file(file): + found_match = False + for filename in os.listdir(file.getAbsolutePath()): + if 'valid-file.txt' == filename: + found_match = True + break + result = [] + if not found_match: + result.append(createFileValidationError("No file named valid-file.txt was found in " + file.getName())) + + return result \ No newline at end of file diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/validation/invalid-data-set/.gitignore b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/validation/invalid-data-set/.gitignore new file mode 100644 index 0000000000000000000000000000000000000000..e69de29bb2d1d6434b8b29ae775ad8c2e48c5391 diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/validation/valid-data-set/valid-file.txt b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/validation/valid-data-set/valid-file.txt new file mode 100644 index 0000000000000000000000000000000000000000..9154070b85fdab23211a5410097222e5d31c8bb4 --- /dev/null +++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/etlserver/validation/valid-data-set/valid-file.txt @@ -0,0 +1 @@ +This is a valid file. \ No newline at end of file diff --git a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/client/api/cli/DssCommandFactoryTest.java b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/client/api/cli/DssCommandFactoryTest.java index f557965bf1679f54522bc7603de133e7bfeb109a..a4f0e5e22ff64d91920c9df17b969a759b614cc1 100644 --- a/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/client/api/cli/DssCommandFactoryTest.java +++ b/datastore_server/sourceTest/java/ch/systemsx/cisd/openbis/dss/client/api/cli/DssCommandFactoryTest.java @@ -78,8 +78,8 @@ public class DssCommandFactoryTest extends AssertJUnit cmd.printUsage(out); String helpText = "usage: dss_client.sh COMMAND [options...] <command arguments>\n" + "\n" - + "Commands:\n" + " ls\n" + " get\n" + " put\n" + "\n" + "Options:\n" - + " [-p,--password] VAL : User login password\n" + + "Commands:\n" + " ls\n" + " get\n" + " put\n" + "testvalid\n" + "\n" + + "Options:\n" + " [-p,--password] VAL : User login password\n" + " [-s,--server-base-url] VAL : URL for openBIS Server (required)\n" + " [-u,--username] VAL : User login name\n"; assertEquals(helpText, bos.toString());