Skip to content
Snippets Groups Projects
Commit cf25f9c3 authored by felmer's avatar felmer
Browse files

LMS-103

First version of file system based storage (without symbolic links) implemented and tested

SVN: 2133
parent f8f02b75
No related branches found
No related tags found
No related merge requests found
Showing
with 369 additions and 71 deletions
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="source/java"/>
<classpathentry kind="src" path="sourceTest/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry combineaccessrules="false" kind="src" path="/common"/>
<classpathentry kind="lib" path="/libraries/commons-io/commons-io.jar" sourcepath="/libraries/commons-io/src.zip"/>
<classpathentry kind="output" path="targets/classes"/>
</classpath>
<?xml version="1.0" encoding="UTF-8"?>
<classpath>
<classpathentry kind="src" path="source/java"/>
<classpathentry kind="src" path="sourceTest/java"/>
<classpathentry kind="con" path="org.eclipse.jdt.launching.JRE_CONTAINER"/>
<classpathentry combineaccessrules="false" kind="src" path="/common"/>
<classpathentry kind="lib" path="/libraries/commons-io/commons-io.jar" sourcepath="/libraries/commons-io/src.zip"/>
<classpathentry kind="lib" path="/libraries/testng/testng-jdk15.jar" sourcepath="/libraries/testng/src.zip"/>
<classpathentry kind="output" path="targets/classes"/>
</classpath>
#
#
#
#show allResults
show allResults
{package} = ch.systemsx.cisd.bds
[storage] = ${package}.storage.* excluding ${package}.storage.*.*
check sets [storage]
[storage.filesystem] = ${package}.storage.filesystem.*
[storage.hdf5] = ${package}.storage.hdf5.*
check sets [storage.filesystem] [storage.hdf5]
[main] = ${package}.* excluding ${package}.*.*
check sets [main]
layer layer0 = [storage]
layer layer1 = [storage.filesystem] [storage.hdf5]
layer layer2 = [main]
check layeringOf layer0 layer1 layer2
......@@ -34,9 +34,9 @@ public class ExperimentIdentifier
public static ExperimentIdentifier loadFrom(IDirectory directory)
{
IDirectory idFolder = Utilities.getSubDirectory(directory, FOLDER);
String groupCode = Utilities.getString(idFolder, GROUP_CODE);
String projectCode = Utilities.getString(idFolder, PROJECT_CODE);
String experimentCode = Utilities.getString(idFolder, EXPERIMENT_CODE);
String groupCode = Utilities.getTrimmedString(idFolder, GROUP_CODE);
String projectCode = Utilities.getTrimmedString(idFolder, PROJECT_CODE);
String experimentCode = Utilities.getTrimmedString(idFolder, EXPERIMENT_CODE);
return new ExperimentIdentifier(groupCode, projectCode, experimentCode);
}
......
......@@ -43,7 +43,7 @@ public enum ProcessingType
public static ProcessingType loadFrom(IDirectory directory)
{
return resolve(Utilities.getString(directory, PROCESSING_TYPE));
return resolve(Utilities.getTrimmedString(directory, PROCESSING_TYPE));
}
public void saveTo(IDirectory directory)
......
......@@ -38,6 +38,11 @@ public class Utilities
return (IDirectory) node;
}
public static String getTrimmedString(IDirectory directory, String name)
{
return getString(directory, name).trim();
}
public static String getString(IDirectory directory, String name)
{
INode node = directory.getNode(name);
......@@ -45,12 +50,12 @@ public class Utilities
{
throw new UserFailureException("File '" + name + "' missing in " + directory);
}
if (node instanceof IFile<?> == false)
if (node instanceof IFile == false)
{
throw new UserFailureException(node + " is not a file.");
}
IFile<?> file = (IFile<?>) node;
return file.getValue().toString();
IFile file = (IFile) node;
return file.getStringContent();
}
}
......@@ -38,7 +38,7 @@ public final class Version
private static int getNumber(IDirectory versionFolder, String name)
{
String value = Utilities.getString(versionFolder, name);
String value = Utilities.getTrimmedString(versionFolder, name);
try
{
return Integer.parseInt(value);
......
......@@ -29,9 +29,9 @@ public interface IDirectory extends INode, Iterable<INode>
public IDirectory makeDirectory(String name);
public IFile<File> addRealFile(File file);
public INode addFile(File file);
public IFile<String> addKeyValuePair(String key, String value);
public IFile addKeyValuePair(String key, String value);
public ILink addLink(String name, INode node);
}
......@@ -21,10 +21,15 @@ package ch.systemsx.cisd.bds.storage;
*
* @author Franz-Josef Elmer
*/
public interface IFile<T> extends INode
public interface IFile extends INode
{
/**
* Returns the value (or content) of this file node.
* Returns the content of this file node as a byte array.
*/
public T getValue();
public byte[] getBinaryContent();
/**
* Returns the content of this file node as a string.
*/
public String getStringContent();
}
......@@ -33,6 +33,7 @@ abstract class AbstractNode implements INode
AbstractNode(File file)
{
assert file != null : "Unspecified file";
assert file.exists() : "Non existing file " + file;
this.nodeFile = file;
}
......
......@@ -16,7 +16,6 @@
package ch.systemsx.cisd.bds.storage.filesystem;
import java.io.File;
import java.util.Iterator;
import ch.systemsx.cisd.bds.storage.IDirectory;
......@@ -28,13 +27,11 @@ import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.utilities.FileUtilities;
/**
*
*
* @author Franz-Josef Elmer
*/
class Directory extends AbstractNode implements IDirectory
{
Directory(File directory)
Directory(java.io.File directory)
{
super(directory);
assert directory.isDirectory() : "Not a directory: " + directory.getAbsolutePath();
......@@ -42,8 +39,8 @@ class Directory extends AbstractNode implements IDirectory
public INode getNode(String name)
{
File[] files = nodeFile.listFiles();
for (File file : files)
java.io.File[] files = nodeFile.listFiles();
for (java.io.File file : files)
{
if (file.getName().equals(name))
{
......@@ -55,10 +52,10 @@ class Directory extends AbstractNode implements IDirectory
public IDirectory makeDirectory(String name)
{
File dir = new File(nodeFile, name);
java.io.File dir = new java.io.File(nodeFile, name);
if (dir.exists())
{
throw new UserFailureException("There already exists a file named '" + name + "' directory " + this);
throw new UserFailureException("There already exists a file named '" + name + "' in directory " + this);
}
boolean successful = dir.mkdir();
if (successful == false)
......@@ -69,18 +66,18 @@ class Directory extends AbstractNode implements IDirectory
return new Directory(dir);
}
public IFile<String> addKeyValuePair(String key, String value)
public IFile addKeyValuePair(String key, String value)
{
File file = new File(nodeFile, key);
java.io.File file = new java.io.File(nodeFile, key);
FileUtilities.writeToFile(file, value);
return new StringFile(file);
return new File(file);
}
public IFile<File> addRealFile(File file)
public INode addFile(java.io.File file)
{
File newFile = new File(nodeFile, file.getName());
FileUtilities.copyFileTo(file, newFile, true);
return new FileFile(newFile);
INode node = NodeFactory.createNode(file);
node.extractTo(nodeFile);
return node;
}
public ILink addLink(String name, INode node)
......@@ -93,7 +90,7 @@ class Directory extends AbstractNode implements IDirectory
{
return new Iterator<INode>()
{
private File[] files = nodeFile.listFiles();
private java.io.File[] files = nodeFile.listFiles();
private int index;
public void remove()
......@@ -113,9 +110,9 @@ class Directory extends AbstractNode implements IDirectory
};
}
public void extractTo(File directory) throws UserFailureException, EnvironmentFailureException
public void extractTo(java.io.File directory) throws UserFailureException, EnvironmentFailureException
{
File destination = new File(directory, getName());
java.io.File destination = new java.io.File(directory, getName());
if (destination.mkdirs() == false)
{
throw new EnvironmentFailureException("Couldn't create directory for some unknown reason: "
......
......@@ -16,7 +16,6 @@
package ch.systemsx.cisd.bds.storage.filesystem;
import java.io.File;
import java.io.FileInputStream;
import java.io.FileOutputStream;
import java.io.IOException;
......@@ -26,33 +25,56 @@ import org.apache.commons.io.IOUtils;
import ch.systemsx.cisd.bds.storage.IFile;
import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.utilities.FileUtilities;
/**
*
*
* @author Franz-Josef Elmer
*/
abstract class AbstractFileNode<T> extends AbstractNode implements IFile<T>
class File extends AbstractNode implements IFile
{
AbstractFileNode(File file)
File(java.io.File file)
{
super(file);
assert file.isFile() : "Not a file " + file.getAbsolutePath();
}
public byte[] getBinaryContent()
{
FileInputStream inputStream = null;
try
{
inputStream = new FileInputStream(nodeFile);
return IOUtils.toByteArray(inputStream);
} catch (IOException ex)
{
throw new EnvironmentFailureException("Couldn't get data from file " + nodeFile.getAbsolutePath(), ex);
} finally
{
IOUtils.closeQuietly(inputStream);
}
}
public String getStringContent()
{
return FileUtilities.loadToString(nodeFile);
}
public void extractTo(File directory) throws UserFailureException, EnvironmentFailureException
public void extractTo(java.io.File directory) throws UserFailureException, EnvironmentFailureException
{
FileInputStream inputStream = null;
FileOutputStream outputStream = null;
try
{
inputStream = new FileInputStream(nodeFile);
outputStream = new FileOutputStream(new File(directory, getName()));
directory.mkdirs();
outputStream = new FileOutputStream(new java.io.File(directory, getName()));
IOUtils.copy(inputStream, outputStream);
} catch (IOException ex)
{
throw new EnvironmentFailureException("Couldn't extract file '" + this + "' to directory "
+ directory.getAbsolutePath());
+ directory.getAbsolutePath(), ex);
} finally
{
IOUtils.closeQuietly(inputStream);
......
......@@ -16,7 +16,6 @@
package ch.systemsx.cisd.bds.storage.filesystem;
import java.io.File;
import java.io.IOException;
import ch.systemsx.cisd.bds.storage.INode;
......@@ -29,7 +28,7 @@ import ch.systemsx.cisd.common.exceptions.EnvironmentFailureException;
*/
class NodeFactory
{
static INode createNode(File file)
static INode createNode(java.io.File file)
{
assert file != null : "Unspecified node";
String absolutePath = file.getAbsolutePath();
......@@ -38,13 +37,13 @@ class NodeFactory
String canonicalPath = file.getCanonicalPath();
if (absolutePath.equals(canonicalPath) == false)
{
return new Link(file.getName(), createNode(new File(canonicalPath)));
return new Link(file.getName(), createNode(new java.io.File(canonicalPath)));
}
if (file.isDirectory())
{
return new Directory(file);
}
return new StringFile(file);
return new File(file);
} catch (IOException ex)
{
throw new EnvironmentFailureException("Couldn't get canonical path of file " + absolutePath);
......
/*
* Copyright 2007 ETH Zuerich, CISD
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.systemsx.cisd.bds.storage.filesystem;
import static org.testng.AssertJUnit.assertEquals;
import static org.testng.AssertJUnit.assertFalse;
import static org.testng.AssertJUnit.assertNotNull;
import static org.testng.AssertJUnit.assertTrue;
import java.io.File;
import org.testng.AssertJUnit;
import org.testng.annotations.Test;
import ch.systemsx.cisd.bds.storage.IDirectory;
import ch.systemsx.cisd.bds.storage.IFile;
import ch.systemsx.cisd.bds.storage.INode;
import ch.systemsx.cisd.common.exceptions.UserFailureException;
import ch.systemsx.cisd.common.utilities.FileUtilities;
/**
*
*
* @author Franz-Josef Elmer
*/
public class DirectoryTest extends StorageTestCase
{
@Test
public void testGetName()
{
Directory directory = new Directory(TEST_DIR);
assertEquals(TEST_DIR.getName(), directory.getName());
}
@Test
public void testMakeDirectory()
{
Directory directory = new Directory(TEST_DIR);
IDirectory subdirectory = directory.makeDirectory("sub-directory");
assertEquals("sub-directory", subdirectory.getName());
File subdir = new File(TEST_DIR, "sub-directory");
assertEquals(true, subdir.exists());
assertEquals(true, subdir.isDirectory());
}
@Test
public void testMakeDirectoryTwice()
{
Directory directory = new Directory(TEST_DIR);
directory.makeDirectory("sub-directory");
try
{
directory.makeDirectory("sub-directory");
AssertJUnit.fail("UserFailureException because a directory can made only once.");
} catch (UserFailureException e)
{
assertTrue(e.getMessage().indexOf("sub-directory") >= 0);
}
}
@Test
public void testAddKeyValuePair()
{
IFile stringFile = new Directory(TEST_DIR).addKeyValuePair("answer", "42");
assertEquals("42\n", FileUtilities.loadToString(new File(TEST_DIR, "answer")));
assertEquals("42\n", stringFile.getStringContent());
}
@Test
public void testExtractTo()
{
File dir = new File(TEST_DIR, "dir");
dir.mkdirs();
Directory directory = new Directory(dir);
directory.addKeyValuePair("p1", "property 1");
IDirectory subdir = directory.makeDirectory("subdir");
subdir.addKeyValuePair("p2", "property 2");
File destination = new File(TEST_DIR, "destination");
assertFalse(destination.exists());
directory.extractTo(destination);
assertTrue(destination.exists());
File copiedDir = new File(destination, "dir");
assertTrue(copiedDir.exists());
assertEquals("property 1\n", FileUtilities.loadToString(new File(copiedDir, "p1")));
File copiedSubDir = new File(copiedDir, "subdir");
assertTrue(copiedSubDir.isDirectory());
assertEquals("property 2\n", FileUtilities.loadToString(new File(copiedSubDir, "p2")));
}
@Test
public void testAddFile()
{
File dir = new File(TEST_DIR, "dir");
dir.mkdirs();
FileUtilities.writeToFile(new File(dir, "p1"), "property 1");
File subDir = new File(dir, "subdir");
subDir.mkdir();
FileUtilities.writeToFile(new File(subDir, "p2"), "property 2");
File dest = new File(TEST_DIR, "destination");
dest.mkdir();
Directory directory = new Directory(dest);
INode copiedDir = directory.addFile(dir);
assertEquals("dir", copiedDir.getName());
assertTrue(copiedDir instanceof IDirectory);
File copiedRealDir = new File(dest, "dir");
assertTrue(copiedRealDir.isDirectory());
IDirectory cd = (IDirectory) copiedDir;
INode node = cd.getNode("p1");
assertNotNull(node);
assertTrue(node instanceof IFile);
assertEquals("property 1\n", ((IFile) node).getStringContent());
assertEquals("property 1\n", FileUtilities.loadToString(new File(copiedRealDir, "p1")));
node = cd.getNode("subdir");
assertEquals("subdir", node.getName());
assertNotNull(node);
assertTrue(node instanceof IDirectory);
node = ((IDirectory) node).getNode("p2");
File copiedRealSubDir = new File(copiedRealDir, "subdir");
assertTrue(copiedRealSubDir.isDirectory());
assertEquals("p2", node.getName());
assertTrue(node instanceof IFile);
assertEquals("property 2\n", ((IFile) node).getStringContent());
assertEquals("property 2\n", FileUtilities.loadToString(new File(copiedRealSubDir, "p2")));
}
}
......@@ -16,23 +16,20 @@
package ch.systemsx.cisd.bds.storage.filesystem;
import java.io.File;
import org.testng.AssertJUnit;
import org.testng.annotations.Test;
/**
*
*
* @author Franz-Josef Elmer
*/
class FileFile extends AbstractFileNode<File>
public class FileStorageTest extends StorageTestCase
{
public FileFile(File file)
@Test
public void testGetRoot()
{
super(file);
FileStorage fileStorage = new FileStorage(TEST_DIR);
AssertJUnit.assertEquals(TEST_DIR.getName(), fileStorage.getRoot().getName());
}
public File getValue()
{
return nodeFile;
}
}
/*
* Copyright 2007 ETH Zuerich, CISD
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.systemsx.cisd.bds.storage.filesystem;
import static org.testng.AssertJUnit.assertEquals;
import org.testng.annotations.Test;
import ch.systemsx.cisd.common.utilities.FileUtilities;
/**
*
*
* @author Franz-Josef Elmer
*/
public class FileTest extends StorageTestCase
{
@Test
public void testGetValueAndGetName()
{
java.io.File file = new java.io.File(TEST_DIR, "test.txt");
FileUtilities.writeToFile(file, "Hello\nworld!\n");
File stringFile = new File(file);
assertEquals("test.txt", stringFile.getName());
assertEquals("Hello\nworld!\n", stringFile.getStringContent());
assertEquals("Hello\nworld!\n", new String(stringFile.getBinaryContent()));
}
@Test
public void testExtractTo()
{
java.io.File file = new java.io.File(TEST_DIR, "test.txt");
FileUtilities.writeToFile(file, "Hello\nworld!\n");
File stringFile = new File(file);
java.io.File subdir = new java.io.File(TEST_DIR, "subdir");
stringFile.extractTo(subdir);
assertEquals("Hello\nworld!\n", FileUtilities.loadToString(new java.io.File(subdir, stringFile.getName())));
}
}
/*
* Copyright 2007 ETH Zuerich, CISD
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ch.systemsx.cisd.bds.storage.filesystem;
import static org.testng.AssertJUnit.assertEquals;
import static org.testng.AssertJUnit.assertTrue;
import org.testng.annotations.Test;
import ch.systemsx.cisd.bds.storage.IDirectory;
import ch.systemsx.cisd.bds.storage.IFile;
import ch.systemsx.cisd.bds.storage.INode;
import ch.systemsx.cisd.common.utilities.FileUtilities;
/**
*
*
* @author Franz-Josef Elmer
*/
public class NodeFactoryTest extends StorageTestCase
{
@Test
public void testCreateFileNode()
{
java.io.File file = new java.io.File(TEST_DIR, "text.txt");
FileUtilities.writeToFile(file, "hello");
INode node = NodeFactory.createNode(file);
assertTrue(node instanceof IFile);
assertEquals("text.txt", node.getName());
}
@Test
public void testCreateDirectoryNode()
{
java.io.File file = new java.io.File(TEST_DIR, "dir");
file.mkdir();
INode node = NodeFactory.createNode(file);
assertTrue(node instanceof IDirectory);
assertEquals("dir", node.getName());
}
}
......@@ -17,24 +17,25 @@
package ch.systemsx.cisd.bds.storage.filesystem;
import java.io.File;
import java.io.IOException;
import ch.systemsx.cisd.common.utilities.FileUtilities;
import org.apache.commons.io.FileUtils;
import org.testng.annotations.BeforeMethod;
/**
*
*
* @author Franz-Josef Elmer
*/
class StringFile extends AbstractFileNode<String>
public class StorageTestCase
{
StringFile(File file)
{
super(file);
}
static final File TEST_DIR = new File("targets" + File.separator + "unit-test-wd");
public String getValue()
@BeforeMethod
public void setup() throws IOException
{
return FileUtilities.loadToString(nodeFile).trim();
TEST_DIR.mkdirs();
FileUtils.cleanDirectory(TEST_DIR);
}
}
0% Loading or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment