Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
postprocessing
Media Converter
Commits
f61b9c84
Unverified
Commit
f61b9c84
authored
Aug 21, 2020
by
Sebastian Schüpbach
Browse files
fix style issues
Signed-off-by:
Sebastian Schüpbach
<
sebastian.schuepbach@unibas.ch
>
parent
9dd10dc6
Pipeline
#13102
failed with stages
in 3 minutes and 9 seconds
Changes
3
Pipelines
1
Hide whitespace changes
Inline
Side-by-side
src/main/scala/ch/memobase/App.scala
View file @
f61b9c84
...
...
@@ -55,12 +55,13 @@ object App extends scala.App with Logging {
settings
.
getAppSettings
.
getProperty
(
"fedoraPassword"
)
)
val
recordProcessor
=
new
RecordProcessor
(
fileHandler
,
fCWrapper
,
settings
.
getAppSettings
.
getProperty
(
"externalBaseUrl"
))
val
consumerPollTimeoutMs
=
100
try
{
logger
.
debug
(
s
"Subscribing to topic ${settings.getInputTopic}"
)
consumer
.
subscribe
(
List
(
settings
.
getInputTopic
).
asJava
)
while
(
true
)
{
val
records
=
consumer
.
poll
(
Duration
.
ofMillis
(
100
)).
asScala
val
records
=
consumer
.
poll
(
Duration
.
ofMillis
(
consumerPollTimeoutMs
)).
asScala
for
(
record
<-
records
)
{
// TODO: Filter for objectTypes: Only records, not institutions, recordSets
recordProcessor
.
process
(
record
)
match
{
...
...
src/main/scala/ch/memobase/FedoraClientWrapper.scala
View file @
f61b9c84
...
...
@@ -25,6 +25,7 @@ import java.net.URI
import
ch.memobase.models._
import
org.memobase.fedora.
{
BinaryResource
,
FedoraClient
,
FedoraClientImpl
}
import
scala.language.postfixOps
import
scala.util.
{
Failure
,
Success
,
Try
}
...
...
@@ -82,9 +83,10 @@ object FedoraClientWrapper {
}
private
def
copyDataAndCloseResource
(
binaryResource
:
BinaryResource
)
:
Try
[
ByteArrayOutputStream
]
=
{
val
dataByteLength
=
2097152
val
outputStream
:
Try
[
ByteArrayOutputStream
]
=
Try
{
val
oS
=
new
ByteArrayOutputStream
()
val
data
=
new
Array
[
Byte
](
2097152
)
val
data
=
new
Array
[
Byte
](
dataByteLength
)
Iterator
.
continually
(
binaryResource
.
getData
.
read
(
data
))
.
takeWhile
(-
1
!=)
.
foreach
(
oS
.
write
)
...
...
src/main/scala/ch/memobase/RecordProcessor.scala
View file @
f61b9c84
...
...
@@ -20,6 +20,8 @@
package
ch.memobase
import
java.io.ByteArrayOutputStream
import
ch.memobase.models._
import
org.apache.kafka.clients.consumer.ConsumerRecord
...
...
@@ -45,7 +47,15 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper
(
for
{
kafkaMsg
<-
BinaryResourceMetadata
.
build
(
record
.
value
(),
externalBaseUrl
)
fileWithMetadata
<-
fedoraClientWrapper
.
fetchBinaryResource
(
kafkaMsg
.
filePath
)
}
yield
(
kafkaMsg
.
id
,
kafkaMsg
.
eventType
,
fileWithMetadata
.
fileType
,
fileWithMetadata
.
data
)
match
{
}
yield
createProcessResult
(
kafkaMsg
.
id
,
kafkaMsg
.
eventType
,
fileWithMetadata
.
fileType
,
fileWithMetadata
.
data
))
.
recover
{
case
e
:
ResourceWithoutBinary
=>
ProcessIgnore
(
record
.
key
(),
e
.
getMessage
)
case
e
:
Exception
=>
ProcessFailure
(
record
.
key
(),
UnknownFileType
,
""
,
e
)
}
}.
get
private
def
createProcessResult
(
id
:
String
,
fedoraEvent
:
Event
,
mediaFileType
:
MediaFileType
,
data
:
ByteArrayOutputStream
)
=
(
id
,
fedoraEvent
,
mediaFileType
,
data
)
match
{
case
(
id
,
Create
|
Update
,
af
:
AudioFileType
,
data
)
=>
errorHandler
(
fileHandler
.
createAudioCopy
(
data
,
id
,
af
),
id
,
af
,
"Create/Update"
)
case
(
id
,
Delete
,
af
:
AudioFileType
,
_
)
=>
...
...
@@ -60,9 +70,5 @@ class RecordProcessor(fileHandler: DisseminationCopyHandler, fedoraClientWrapper
errorHandler
(
fileHandler
.
deleteImageCopy
(
id
),
id
,
i
,
"Delete"
)
case
(
id
,
event
,
ft
,
_
)
=>
ProcessFailure
(
id
,
ft
,
event
.
toString
,
new
Exception
)
}).
recover
{
case
e
:
ResourceWithoutBinary
=>
ProcessIgnore
(
record
.
key
(),
e
.
getMessage
)
case
e
:
Exception
=>
ProcessFailure
(
record
.
key
(),
UnknownFileType
,
""
,
e
)
}
}.
get
}
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment