Skip to content
GitLab
Explore
Sign in
Primary navigation
Search or go to…
Project
O
openbis
Manage
Activity
Members
Labels
Code
Merge requests
Repository
Branches
Commits
Tags
Repository graph
Compare revisions
Snippets
Build
Pipelines
Jobs
Pipeline schedules
Artifacts
Deploy
Releases
Model registry
Analyze
Value stream analytics
Contributor analytics
CI/CD analytics
Repository analytics
Help
Help
Support
GitLab documentation
Compare GitLab plans
Community forum
Contribute to GitLab
Provide feedback
Keyboard shortcuts
?
Snippets
Groups
Projects
Show more breadcrumbs
sispub
openbis
Commits
705f640c
Commit
705f640c
authored
6 years ago
by
felmer
Browse files
Options
Downloads
Patches
Plain Diff
SSDM-7430: Bug in DataSetRegistrationIngestionService.downloadDataSetFiles() fixed
parent
50d82489
No related branches found
Branches containing commit
No related tags found
Tags containing commit
No related merge requests found
Changes
1
Hide whitespace changes
Inline
Side-by-side
Showing
1 changed file
datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationIngestionService.java
+21
-20
21 additions, 20 deletions
...llelizedExecutor/DataSetRegistrationIngestionService.java
with
21 additions
and
20 deletions
datastore_server/source/java/ch/ethz/sis/openbis/generic/server/dss/plugins/sync/harvester/synchronizer/parallelizedExecutor/DataSetRegistrationIngestionService.java
+
21
−
20
View file @
705f640c
...
@@ -17,11 +17,10 @@
...
@@ -17,11 +17,10 @@
package
ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.parallelizedExecutor
;
package
ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.parallelizedExecutor
;
import
java.io.File
;
import
java.io.File
;
import
java.io.FileOutputStream
;
import
java.io.IOException
;
import
java.io.InputStream
;
import
java.io.InputStream
;
import
java.io.OutputStream
;
import
java.io.OutputStream
;
import
java.nio.file.Files
;
import
java.nio.file.Path
;
import
java.nio.file.Paths
;
import
java.util.HashMap
;
import
java.util.HashMap
;
import
java.util.LinkedList
;
import
java.util.LinkedList
;
import
java.util.List
;
import
java.util.List
;
...
@@ -42,6 +41,7 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId;
...
@@ -42,6 +41,7 @@ import ch.ethz.sis.openbis.generic.dssapi.v3.dto.datasetfile.id.IDataSetFileId;
import
ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig
;
import
ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.config.SyncConfig
;
import
ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.DSPropertyUtils
;
import
ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.DSPropertyUtils
;
import
ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.V3Facade
;
import
ch.ethz.sis.openbis.generic.server.dss.plugins.sync.harvester.synchronizer.util.V3Facade
;
import
ch.systemsx.cisd.base.exceptions.CheckedExceptionTunnel
;
import
ch.systemsx.cisd.common.io.IOUtilities
;
import
ch.systemsx.cisd.common.io.IOUtilities
;
import
ch.systemsx.cisd.common.logging.LogCategory
;
import
ch.systemsx.cisd.common.logging.LogCategory
;
import
ch.systemsx.cisd.common.logging.LogFactory
;
import
ch.systemsx.cisd.common.logging.LogFactory
;
...
@@ -202,7 +202,7 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
...
@@ -202,7 +202,7 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
}
}
}
}
private
void
downloadDataSetFiles
(
File
dir
,
String
dataSetCode
)
throws
Exception
private
void
downloadDataSetFiles
(
File
dir
,
String
dataSetCode
)
{
{
V3Facade
v3FacadeToDataSource
=
new
V3Facade
(
config
);
V3Facade
v3FacadeToDataSource
=
new
V3Facade
(
config
);
DataSetFileFetchOptions
dsFileFetchOptions
=
new
DataSetFileFetchOptions
();
DataSetFileFetchOptions
dsFileFetchOptions
=
new
DataSetFileFetchOptions
();
...
@@ -226,30 +226,31 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
...
@@ -226,30 +226,31 @@ class DataSetRegistrationIngestionService extends IngestionService<DataSetInform
{
{
DataSetFile
orgFile
=
fileDownload
.
getDataSetFile
();
DataSetFile
orgFile
=
fileDownload
.
getDataSetFile
();
if
(
orgFile
.
getPath
().
equals
(
""
))
if
(
orgFile
.
getPath
().
equals
(
""
))
{
continue
;
continue
;
// if (dsFile.getPath().equals("original"))
}
// continue;
File
output
=
new
File
(
dir
,
orgFile
.
getPath
());
String
filePath
=
orgFile
.
getPath
();
// .substring("original/".length());
File
output
=
new
File
(
dir
.
getAbsolutePath
(),
filePath
);
if
(
orgFile
.
isDirectory
())
if
(
orgFile
.
isDirectory
())
{
{
output
.
mkdirs
();
output
.
mkdirs
();
}
}
else
else
{
{
DataSetFilePermId
filePermId
=
orgFile
.
getPermId
();
DataSetFilePermId
filePermId
=
orgFile
.
getPermId
();
FileDetails
fileDetails
=
fileDetailsMap
.
get
(
filePermId
);
FileDetails
fileDetails
=
fileDetailsMap
.
get
(
filePermId
);
Path
path
=
Paths
.
get
(
dir
.
getAbsolutePath
(),
filePath
);
output
.
getParentFile
().
mkdirs
();
InputStream
inputStream
=
fileDownload
.
getInputStream
();
try
(
OutputStream
outputStream
=
new
FileOutputStream
(
output
))
OutputStream
outputStream
=
Files
.
newOutputStream
(
path
);
{
int
checksumCRC32
=
IOUtilities
.
copyAndGetChecksumCRC32
(
inputStream
,
outputStream
);
int
checksumCRC32
=
IOUtilities
.
copyAndGetChecksumCRC32
(
fileDownload
.
getInputStream
(),
outputStream
);
File
copiedFile
=
new
File
(
path
.
normalize
().
toString
());
if
(
checksumCRC32
!=
fileDetails
.
getCrc32checksum
()
if
(
checksumCRC32
!=
fileDetails
.
getCrc32checksum
()
||
output
.
length
()
!=
fileDetails
.
getFileLength
())
||
copiedFile
.
length
()
!=
fileDetails
.
getFileLength
())
{
throw
new
RuntimeException
(
"Crc32 or file length does not match for "
+
orgFile
.
getPath
()
+
" calculated:"
+
checksumCRC32
+
" expected:"
+
fileDetails
.
getCrc32checksum
());
}
}
catch
(
IOException
e
)
{
{
throw
new
RuntimeException
(
"Crc32 or file length does not match for "
+
orgFile
.
getPath
()
+
" calculated:"
+
checksumCRC32
throw
CheckedExceptionTunnel
.
wrapIfNecessary
(
e
);
+
" expected:"
+
fileDetails
.
getCrc32checksum
());
}
}
}
}
}
}
...
...
This diff is collapsed.
Click to expand it.
Preview
0%
Loading
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Save comment
Cancel
Please
register
or
sign in
to comment