Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Import Process
Media-Linker
Commits
1650341f
Unverified
Commit
1650341f
authored
Nov 20, 2020
by
Sebastian Schüpbach
Browse files
improve reporting
parent
d7658413
Pipeline
#17573
passed with stages
in 4 minutes and 18 seconds
Changes
5
Pipelines
2
Hide whitespace changes
Inline
Side-by-side
src/main/kotlin/KafkaTopology.kt
View file @
1650341f
...
...
@@ -18,6 +18,10 @@
package
org.memobase
import
java.io.StringReader
import
java.io.StringWriter
import
java.net.MalformedURLException
import
java.net.URL
import
org.apache.jena.rdf.model.Model
import
org.apache.jena.rdf.model.ModelFactory
import
org.apache.jena.rdf.model.Resource
...
...
@@ -35,10 +39,6 @@ import org.memobase.settings.SettingsLoader
import
org.memobase.sftp.SftpClient
import
settings.HeaderExtractionTransformSupplier
import
settings.HeaderMetadata
import
java.io.StringReader
import
java.io.StringWriter
import
java.net.MalformedURLException
import
java.net.URL
class
KafkaTopology
(
private
val
settings
:
SettingsLoader
)
{
private
val
appSettings
=
settings
.
appSettings
...
...
@@ -227,7 +227,11 @@ class KafkaTopology(private val settings: SettingsLoader) {
addLocalThumbnail
(
enrichedValue
,
recordResource
,
digitalObjectResource
,
filePath
)
}
else
{
log
.
warn
(
"No thumbnail url available for ${value.third.id}"
)
enrichedValue
updateRecord
(
enrichedValue
,
ReportStatus
.
warning
,
thumbnailMessage
=
"Download of youtube / vimeo thumbnail failed. Check if resource is still available."
)
}
}
}
...
...
@@ -363,7 +367,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
val
originalIdentifierValue
=
getOriginalIdentifier
(
recordResource
)
!!
val
link
=
getLinkToResourceOnSFTPServer
(
data
.
first
.
second
.
recordSetId
,
type
,
originalIdentifierValue
)
?:
return
if
(
type
==
Constant
.
thumbnailFolderName
)
{
updateRecord
(
data
,
ReportStatus
.
success
,
thumbnailMessage
=
"no local thumbnails available"
)
updateRecord
(
data
,
ReportStatus
.
ignore
,
thumbnailMessage
=
"no local thumbnails available"
)
}
else
{
updateRecord
(
data
,
ReportStatus
.
warning
,
digitalObjectMessage
=
ReportMessages
.
reportFailure
(
key
,
type
))
}
...
...
src/main/kotlin/RemoteResourceHandler.kt
View file @
1650341f
package
org.memobase
import
org.apache.logging.log4j.LogManager
import
org.memobase.exceptions.SftpClientException
import
org.memobase.sftp.SftpClient
import
java.io.ByteArrayOutputStream
import
java.io.FileNotFoundException
import
java.io.FileOutputStream
...
...
@@ -13,6 +10,9 @@ import java.net.URLEncoder
import
java.nio.charset.StandardCharsets
import
java.nio.file.Files
import
java.nio.file.Paths
import
org.apache.logging.log4j.LogManager
import
org.memobase.exceptions.SftpClientException
import
org.memobase.sftp.SftpClient
/**
* Queries oembed API for videos on Vimeo or Youtube and downloads poster images
...
...
src/test/kotlin/TestKafkaTopology.kt
View file @
1650341f
...
...
@@ -18,6 +18,12 @@
package
org.memobase
import
com.beust.klaxon.Klaxon
import
java.io.File
import
java.io.FileInputStream
import
java.io.FileOutputStream
import
java.nio.charset.Charset
import
java.nio.file.Paths
import
java.util.stream.Stream
import
org.apache.jena.rdf.model.ModelFactory
import
org.apache.jena.riot.Lang
import
org.apache.jena.riot.RDFDataMgr
...
...
@@ -35,12 +41,6 @@ import org.junit.jupiter.params.ParameterizedTest
import
org.junit.jupiter.params.provider.MethodSource
import
org.memobase.rdf.NS
import
org.memobase.testing.EmbeddedSftpServer
import
java.io.File
import
java.io.FileInputStream
import
java.io.FileOutputStream
import
java.nio.charset.Charset
import
java.nio.file.Paths
import
java.util.stream.Stream
@TestInstance
(
TestInstance
.
Lifecycle
.
PER_CLASS
)
class
TestKafkaTopology
{
...
...
src/test/resources/data/output4.json
View file @
1650341f
{
"id"
:
"https://memobase.ch/record/Tanzarchiv-42858-43349"
,
"status"
:
"
SUCCESS
"
,
"status"
:
"
IGNORE
"
,
"message"
:
"No valid file found for id https://memobase.ch/digital/Tanzarchiv-42858-43349-1 in folder thumbnails."
}
\ No newline at end of file
src/test/resources/data/turtle-output4.ttl
View file @
1650341f
...
...
@@ -44,10 +44,6 @@ mbr:Tanzarchiv-42858-43349
rico:
type
"main"
]
;
rico:
heldBy
mbcb:
Tanzarchiv
;
rico:
identifiedBy
[
a
rico:
Identifier
;
rico:
identifier
"43349"
;
rico:
type
"original"
]
;
rico:
identifiedBy
[
a
rico:
Identifier
;
rico:
identifier
"Tanzarchiv-42858-43349"
;
rico:
type
"main"
...
...
@@ -56,6 +52,10 @@ mbr:Tanzarchiv-42858-43349
rico:
identifier
"Tanzarchiv-43349"
;
rico:
type
"oldMemobase"
]
;
rico:
identifiedBy
[
a
rico:
Identifier
;
rico:
identifier
"43349"
;
rico:
type
"original"
]
;
rico:
isPartOf
mbrs:
Tanzarchiv-42858
;
rico:
recordResourceOrInstantiationIsSourceOfCreationRelation
_:
b0
,
_:
b1
,
_:
b2
;
...
...
@@ -78,16 +78,6 @@ _:b1 a rico:CreationRelation ;
rico:
name
"Komponisten / Compositeur"
;
rico:
type
"contributor"
.
_:
b2
a
rico:
CreationRelation
;
rico:
creationRelationHasSource
mbr:
Tanzarchiv-42858-43349
;
rico:
creationRelationHasTarget
[
a
rico:
Person
;
rico:
agentIsTargetOfCreationRelation
_:
b2
;
rico:
name
"Katarzyna Gdaniec (Choreograf / Chorégraphe)"
]
;
rico:
name
"Author"
;
rico:
type
"creator"
.
mbdo:
Tanzarchiv-42858-43349-1
a
rico:
Instantiation
;
ebucore:
height
"360"
;
...
...
@@ -103,6 +93,26 @@ mbdo:Tanzarchiv-42858-43349-1
mbpo:
Tanzarchiv-42858-43349-1
;
rico:
type
"digitalObject"
.
_:
b2
a
rico:
CreationRelation
;
rico:
creationRelationHasSource
mbr:
Tanzarchiv-42858-43349
;
rico:
creationRelationHasTarget
[
a
rico:
Person
;
rico:
agentIsTargetOfCreationRelation
_:
b2
;
rico:
name
"Katarzyna Gdaniec (Choreograf / Chorégraphe)"
]
;
rico:
name
"Author"
;
rico:
type
"creator"
.
_:
b0
a
rico:
CreationRelation
;
rico:
creationRelationHasSource
mbr:
Tanzarchiv-42858-43349
;
rico:
creationRelationHasTarget
[
a
rico:
CorporateBody
;
rico:
agentIsTargetOfCreationRelation
_:
b0
;
rico:
name
"Compagnie Linga"
]
;
rico:
name
"Kompanie / Compagnie"
;
rico:
type
"contributor"
.
mbpo:
Tanzarchiv-42858-43349-1
a
rico:
Instantiation
;
rdau:
P60558
"Farbig / Couleur"
;
...
...
@@ -134,13 +144,3 @@ mbpo:Tanzarchiv-42858-43349-1
rico:
isDerivedFromInstantiation
mbdo:
Tanzarchiv-42858-43349-1
;
rico:
type
"thumbnail"
.
_:
b0
a
rico:
CreationRelation
;
rico:
creationRelationHasSource
mbr:
Tanzarchiv-42858-43349
;
rico:
creationRelationHasTarget
[
a
rico:
CorporateBody
;
rico:
agentIsTargetOfCreationRelation
_:
b0
;
rico:
name
"Compagnie Linga"
]
;
rico:
name
"Kompanie / Compagnie"
;
rico:
type
"contributor"
.
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment