Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Import Process
Media-Linker
Commits
04ed4b94
Commit
04ed4b94
authored
Sep 10, 2020
by
Jonas Waeber
Browse files
[WIP] First changes to read thumbnails
parent
c44bdf5e
Changes
12
Hide whitespace changes
Inline
Side-by-side
build.gradle
View file @
04ed4b94
...
...
@@ -47,7 +47,7 @@ dependencies {
// JSON Parser
implementation
'com.beust:klaxon:5.2'
implementation
'org.memobase:memobase-service-utilities:
1
.1
2.1
'
implementation
'org.memobase:memobase-service-utilities:
0
.1
4.2
'
implementation
'ch.memobase:import-process-effects-registry_2.12:0.2.1'
implementation
'org.apache.jena:apache-jena:3.14.0'
...
...
helm-charts/templates/app-config.yml
View file @
04ed4b94
...
...
@@ -7,6 +7,7 @@ data:
APPLICATION_ID
:
"
{{
.Values.processId
}}-{{
.Values.jobName
}}"
RECORD_SET_ID
:
"
{{
.Values.recordSetId
}}"
SFTP_BASE_PATH
:
"
{{
.Values.sftpBasePath
}}"
HAS_THUMBNAILS
:
"
{{
.Values.hasThumbnails
}}"
TOPIC_IN
:
"
{{
.Values.processId
}}-{{
.Values.lastJobName
}}"
TOPIC_OUT
:
"
{{
.Values.processId
}}-{{
.Values.jobName
}}"
TOPIC_PROCESS
:
"
{{
.Values.processId
}}-reporting"
\ No newline at end of file
helm-charts/values.yaml
View file @
04ed4b94
...
...
@@ -20,4 +20,5 @@ processId: p0001
institutionId
:
placeholder
recordSetId
:
placeholder
sftpBasePath
:
/swissbib_index/mb_sftp
\ No newline at end of file
sftpBasePath
:
/swissbib_index/mb_sftp
hasThumbnails
:
false
\ No newline at end of file
src/main/kotlin/Constant.kt
View file @
04ed4b94
...
...
@@ -2,9 +2,11 @@ package org.memobase
object
Constant
{
const
val
mediaFolderName
=
"media"
const
val
thumbnailFolderName
=
"thumbnails"
const
val
recordSetIdPropertyName
=
"recordSetId"
const
val
sftpBasePathPropertyName
=
"sftp.basePath"
const
val
topicReportingSuffix
=
"reporting"
const
val
hasThumbnailsPropertyName
=
"hasThumbnails"
const
val
rdfParserLang
=
"NTRIPLES"
...
...
src/main/kotlin/FolderNotFoundException.kt
0 → 100644
View file @
04ed4b94
package
org.memobase
class
FolderNotFoundException
(
folder
:
String
)
:
Exception
(
"Folder $folder does not exist on sftp server."
)
\ No newline at end of file
src/main/kotlin/KafkaTopology.kt
View file @
04ed4b94
...
...
@@ -18,13 +18,9 @@
package
org.memobase
import
java.io.StringReader
import
java.io.StringWriter
import
kotlin.system.exitProcess
import
org.apache.jena.rdf.model.Model
import
org.apache.jena.rdf.model.ModelFactory
import
org.apache.jena.rdf.model.Resource
import
org.apache.jena.rdf.model.Statement
import
org.apache.jena.rdf.model.impl.StatementImpl
import
org.apache.kafka.streams.StreamsBuilder
import
org.apache.kafka.streams.kstream.Predicate
...
...
@@ -36,12 +32,23 @@ import org.memobase.reports.ReportMessages
import
org.memobase.reports.ReportStatus
import
org.memobase.settings.SettingsLoader
import
org.memobase.sftp.SftpClient
import
java.io.StringReader
import
java.io.StringWriter
import
kotlin.system.exitProcess
class
KafkaTopology
(
private
val
settings
:
SettingsLoader
)
{
private
val
log
=
LogManager
.
getLogger
(
"MediaLinker"
)
private
val
appSettings
=
settings
.
appSettings
private
val
fileFetcher
=
LocatorFetcher
(
settings
.
sftpSettings
,
appSettings
.
getProperty
(
Constant
.
sftpBasePathPropertyName
),
appSettings
.
getProperty
(
Constant
.
recordSetIdPropertyName
),
appSettings
.
getProperty
(
Constant
.
hasThumbnailsPropertyName
)
!!
.
toBoolean
()
)
private
val
sftpClient
=
SftpClient
(
settings
.
sftpSettings
)
private
val
sftpBasePath
=
settings
.
appSettings
.
getProperty
(
Constant
.
sftpBasePathPropertyName
)
private
val
sftpBasePath
=
appSettings
.
getProperty
(
Constant
.
sftpBasePathPropertyName
)
private
val
files
=
createMediaFileList
(
settings
.
appSettings
.
getProperty
(
Constant
.
recordSetIdPropertyName
))
...
...
src/main/kotlin/
rdf/EBUCORE
.kt
→
src/main/kotlin/
LocatorFetcher
.kt
View file @
04ed4b94
/*
*
mapper-service
*
Media Linker
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
...
...
@@ -15,30 +15,43 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package
org.memobase
package
org.memobase.rdf
import
org.apache.log4j.LogManager
import
org.memobase.settings.SftpSettings
import
org.memobase.sftp.SftpClient
import
org.apache.jena.rdf.model.Property
import
org.apache.jena.rdf.model.Resource
import
org.apache.jena.rdf.model.ResourceFactory
class
LocatorFetcher
(
sftpSettings
:
SftpSettings
,
basePath
:
String
,
path
:
String
,
private
val
hasThumbnails
:
Boolean
)
{
private
val
log
=
LogManager
.
getLogger
(
"LocatorFetcher"
)
object
EBUCORE
{
private
val
client
=
SftpClient
(
sftpSettings
)
private
val
pathMedia
=
"$basePath/$path/${Constant.mediaFolderName}"
private
val
pathThumbnails
=
"$basePath/$path/${Constant.thumbnailFolderName}"
val
hasGenre
=
prop
(
"hasGenre"
)
val
hasFormat
=
prop
(
"hasFormat"
)
val
hasMedium
=
prop
(
"hasMedium"
)
val
duration
=
prop
(
"duration"
)
val
displayAspectRatio
=
prop
(
"displayAspectRatio"
)
val
audioTrackConfiguration
=
prop
(
"audioTrackConfiguration"
)
val
playbackSpeed
=
prop
(
"playbackSpeed"
)
val
hasStandard
=
prop
(
"hasStandard"
)
val
locator
=
prop
(
"locator"
)
fun
thumbnailFiles
():
List
<
String
>
{
return
if
(
hasThumbnails
)
{
if
(!
client
.
exists
(
pathThumbnails
))
{
throw
FolderNotFoundException
(
pathThumbnails
)
}
log
.
info
(
"Found thumbnail folder on sftp server: $pathThumbnails."
)
client
.
listFiles
(
pathThumbnails
)
}
else
{
emptyList
()
}
}
private
fun
prop
(
name
:
String
):
Property
{
return
ResourceFactory
.
createProperty
(
NS
.
ebucore
,
name
)
fun
mediaFiles
():
List
<
String
>
{
return
if
(
client
.
exists
(
pathMedia
))
{
log
.
info
(
"Found media folder on sftp server: $pathMedia"
)
client
.
listFiles
(
pathMedia
)
}
else
{
throw
FolderNotFoundException
(
pathMedia
)
}
}
private
fun
res
(
name
:
String
):
Resource
{
return
ResourceFactory
.
createResource
(
NS
.
ebucore
+
name
)
fun
fetchThumbnails
()
{
}
}
}
\ No newline at end of file
src/main/kotlin/Service.kt
View file @
04ed4b94
...
...
@@ -33,7 +33,8 @@ class Service(file: String = "app.yml") {
val
settings
=
SettingsLoader
(
listOf
(
Constant
.
recordSetIdPropertyName
,
Constant
.
sftpBasePathPropertyName
Constant
.
sftpBasePathPropertyName
,
Constant
.
hasThumbnailsPropertyName
),
file
,
useStreamsConfig
=
true
,
...
...
src/main/kotlin/rdf/NS.kt
deleted
100644 → 0
View file @
c44bdf5e
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package
org.memobase.rdf
object
NS
{
val
rdf
=
"http://www.w3.org/1999/02/22-rdf-syntax-ns#"
val
rdfs
=
"http://www.w3.org/2000/01/rdf-schema#"
val
owl
=
"http://www.w3.org/2002/07/owl#"
val
skos
=
"http://www.w3.org/2004/02/skos/core#"
val
rico
=
"https://www.ica.org/standards/RiC/ontology#"
val
ebucore
=
"http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#"
val
memint
=
"https://memobase.ch/institution/"
val
memrs
=
"https://memobase.ch/recordSet/"
val
memr
=
"https://memobase.ch/record/"
val
mempo
=
"https://memobase.ch/instantiation/physical/"
val
memdo
=
"https://memobase.ch/instantiation/digital/"
val
dce
=
"http://purl.org/dc/elements/1.1/"
val
dct
=
"http://purl.org/dc/terms/"
val
schema
=
"http://schema.org/"
val
foaf
=
"http://xmlns.com/foaf/0.1/"
val
wdt
=
"http://www.wikidata.org/prop/direct/"
val
wdtn
=
"http://www.wikidata.org/prop/direct-normalized/"
val
rdau
=
"http://rdaregistry.info/Elements/u/"
}
src/main/kotlin/rdf/RDF.kt
deleted
100644 → 0
View file @
c44bdf5e
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package
org.memobase.rdf
import
org.apache.jena.rdf.model.Property
import
org.apache.jena.rdf.model.Resource
import
org.apache.jena.rdf.model.ResourceFactory
object
RDF
{
val
type
:
Property
=
prop
(
"type"
)
private
fun
prop
(
name
:
String
):
Property
{
return
ResourceFactory
.
createProperty
(
NS
.
rdf
,
name
)
}
private
fun
res
(
name
:
String
):
Resource
{
return
ResourceFactory
.
createResource
(
NS
.
rdf
+
name
)
}
}
src/main/kotlin/rdf/RICO.kt
deleted
100644 → 0
View file @
c44bdf5e
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package
org.memobase.rdf
import
org.apache.jena.rdf.model.Property
import
org.apache.jena.rdf.model.Resource
import
org.apache.jena.rdf.model.ResourceFactory
object
RICO
{
// Classes
val
Record
=
res
(
"Record"
)
val
Instantiation
=
res
(
"Instantiation"
)
val
Title
=
res
(
"Title"
)
val
Language
=
res
(
"Language"
)
val
Identifier
=
res
(
"Identifier"
)
val
Place
=
res
(
"Place"
)
val
Agent
=
res
(
"Agent"
)
val
Person
=
res
(
"Person"
)
val
CorporateBody
=
res
(
"CorporateBody"
)
val
Rule
=
res
(
"Rule"
)
val
CreationRelation
=
res
(
"CreationRelation"
)
val
SingleDate
=
res
(
"SingleDate"
)
val
DateRange
=
res
(
"DateRange"
)
val
DateSet
=
res
(
"DateSet"
)
// datatype properties
val
title
:
Property
=
prop
(
"title"
)
val
source
:
Property
=
prop
(
"source"
)
val
descriptiveNote
:
Property
=
prop
(
"descriptiveNote"
)
val
name
:
Property
=
prop
(
"name"
)
val
type
:
Property
=
prop
(
"type"
)
val
expressedDate
:
Property
=
prop
(
"expressedDate"
)
val
normalizedDateValue
:
Property
=
prop
(
"normalizedDateValue"
)
val
normalizedValue
:
Property
=
prop
(
"normalizedValue"
)
val
scopeAndContent
:
Property
=
prop
(
"scopeAndContent"
)
val
physicalCharacteristics
:
Property
=
prop
(
"physicalCharacteristics"
)
// Object Properties
val
hasTitle
:
Property
=
prop
(
"hasTitle"
)
val
isTitleOf
:
Property
=
prop
(
"isTitleOf"
)
val
hasLanguage
:
Property
=
prop
(
"hasLanguage"
)
val
isLanguageOf
:
Property
=
prop
(
"isLanguageOf"
)
val
identifiedBy
:
Property
=
prop
(
"identifiedBy"
)
val
identifies
:
Property
=
prop
(
"identifies"
)
val
identifier
:
Property
=
prop
(
"identifier"
)
val
hasSubject
:
Property
=
prop
(
"hasSubject"
)
val
publishedBy
:
Property
=
prop
(
"publishedBy"
)
val
heldBy
:
Property
=
prop
(
"heldBy"
)
val
isPartOf
:
Property
=
prop
(
"isPartOf"
)
val
hasProvenance
:
Property
=
prop
(
"hasProvenance"
)
val
instantiates
:
Property
=
prop
(
"instantiates"
)
val
hasInstantiation
:
Property
=
prop
(
"hasInstantiation"
)
val
hasDerivedInstantiation
:
Property
=
prop
(
"hasDerivedInstantiation"
)
val
isDerivedFromInstantiation
:
Property
=
prop
(
"isDerivedFromInstantiation"
)
val
creationRelationHasSource
:
Property
=
prop
(
"creationRelationHasSource"
)
val
creationRelationHasTarget
:
Property
=
prop
(
"creationRelationHasTarget"
)
val
recordResourceOrInstantiationIsSourceOfCreationRelation
:
Property
=
prop
(
"recordResourceOrInstantiationIsSourceOfCreationRelation"
)
val
regulates
:
Property
=
prop
(
"regulates"
)
val
intellectualPropertyRightsHeldBy
:
Property
=
prop
(
"intellectualPropertyRightsHeldBy"
)
val
regulatedBy
:
Property
=
prop
(
"regulatedBy"
)
private
fun
prop
(
name
:
String
):
Property
{
return
ResourceFactory
.
createProperty
(
NS
.
rico
,
name
)
}
private
fun
res
(
name
:
String
):
Resource
{
return
ResourceFactory
.
createResource
(
NS
.
rico
+
name
)
}
}
src/main/resources/app.yml
View file @
04ed4b94
...
...
@@ -5,6 +5,7 @@ sftp:
password
:
${SFTP_PASSWORD:?system}
app
:
recordSetId
:
${RECORD_SET_ID:?system}
hasThumbnails
:
${HAS_THUMBNAILS:?system}
sftp
:
basePath
:
${SFTP_BASE_PATH:?system}
kafka
:
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment