Commit 04ed4b94 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

[WIP] First changes to read thumbnails

parent c44bdf5e
......@@ -47,7 +47,7 @@ dependencies {
// JSON Parser
implementation 'com.beust:klaxon:5.2'
implementation 'org.memobase:memobase-service-utilities:1.12.1'
implementation 'org.memobase:memobase-service-utilities:0.14.2'
implementation 'ch.memobase:import-process-effects-registry_2.12:0.2.1'
implementation 'org.apache.jena:apache-jena:3.14.0'
......
......@@ -7,6 +7,7 @@ data:
APPLICATION_ID: "{{ .Values.processId }}-{{ .Values.jobName }}"
RECORD_SET_ID: "{{ .Values.recordSetId }}"
SFTP_BASE_PATH: "{{ .Values.sftpBasePath }}"
HAS_THUMBNAILS: "{{ .Values.hasThumbnails }}"
TOPIC_IN: "{{ .Values.processId }}-{{ .Values.lastJobName }}"
TOPIC_OUT: "{{ .Values.processId }}-{{ .Values.jobName }}"
TOPIC_PROCESS: "{{ .Values.processId }}-reporting"
\ No newline at end of file
......@@ -20,4 +20,5 @@ processId: p0001
institutionId: placeholder
recordSetId: placeholder
sftpBasePath: /swissbib_index/mb_sftp
\ No newline at end of file
sftpBasePath: /swissbib_index/mb_sftp
hasThumbnails: false
\ No newline at end of file
......@@ -2,9 +2,11 @@ package org.memobase
object Constant {
const val mediaFolderName = "media"
const val thumbnailFolderName ="thumbnails"
const val recordSetIdPropertyName = "recordSetId"
const val sftpBasePathPropertyName = "sftp.basePath"
const val topicReportingSuffix = "reporting"
const val hasThumbnailsPropertyName = "hasThumbnails"
const val rdfParserLang = "NTRIPLES"
......
package org.memobase
class FolderNotFoundException(folder: String) : Exception("Folder $folder does not exist on sftp server.")
\ No newline at end of file
......@@ -18,13 +18,9 @@
package org.memobase
import java.io.StringReader
import java.io.StringWriter
import kotlin.system.exitProcess
import org.apache.jena.rdf.model.Model
import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.Statement
import org.apache.jena.rdf.model.impl.StatementImpl
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.kstream.Predicate
......@@ -36,12 +32,23 @@ import org.memobase.reports.ReportMessages
import org.memobase.reports.ReportStatus
import org.memobase.settings.SettingsLoader
import org.memobase.sftp.SftpClient
import java.io.StringReader
import java.io.StringWriter
import kotlin.system.exitProcess
class KafkaTopology(private val settings: SettingsLoader) {
private val log = LogManager.getLogger("MediaLinker")
private val appSettings = settings.appSettings
private val fileFetcher = LocatorFetcher(
settings.sftpSettings,
appSettings.getProperty(Constant.sftpBasePathPropertyName),
appSettings.getProperty(Constant.recordSetIdPropertyName),
appSettings.getProperty(Constant.hasThumbnailsPropertyName)!!.toBoolean()
)
private val sftpClient = SftpClient(settings.sftpSettings)
private val sftpBasePath = settings.appSettings.getProperty(Constant.sftpBasePathPropertyName)
private val sftpBasePath = appSettings.getProperty(Constant.sftpBasePathPropertyName)
private val files = createMediaFileList(settings.appSettings.getProperty(Constant.recordSetIdPropertyName))
......
/*
* mapper-service
* Media Linker
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
......@@ -15,30 +15,43 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
package org.memobase.rdf
import org.apache.log4j.LogManager
import org.memobase.settings.SftpSettings
import org.memobase.sftp.SftpClient
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
class LocatorFetcher(sftpSettings: SftpSettings, basePath: String, path: String, private val hasThumbnails: Boolean) {
private val log = LogManager.getLogger("LocatorFetcher")
object EBUCORE {
private val client = SftpClient(sftpSettings)
private val pathMedia = "$basePath/$path/${Constant.mediaFolderName}"
private val pathThumbnails = "$basePath/$path/${Constant.thumbnailFolderName}"
val hasGenre = prop("hasGenre")
val hasFormat = prop("hasFormat")
val hasMedium = prop("hasMedium")
val duration = prop("duration")
val displayAspectRatio = prop("displayAspectRatio")
val audioTrackConfiguration = prop("audioTrackConfiguration")
val playbackSpeed = prop("playbackSpeed")
val hasStandard = prop("hasStandard")
val locator = prop("locator")
fun thumbnailFiles(): List<String> {
return if (hasThumbnails) {
if (!client.exists(pathThumbnails)) {
throw FolderNotFoundException(pathThumbnails)
}
log.info("Found thumbnail folder on sftp server: $pathThumbnails.")
client.listFiles(pathThumbnails)
} else {
emptyList()
}
}
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.ebucore, name)
fun mediaFiles(): List<String> {
return if (client.exists(pathMedia)) {
log.info("Found media folder on sftp server: $pathMedia")
client.listFiles(pathMedia)
} else {
throw FolderNotFoundException(pathMedia)
}
}
private fun res(name: String): Resource {
return ResourceFactory.createResource(NS.ebucore + name)
fun fetchThumbnails() {
}
}
}
\ No newline at end of file
......@@ -33,7 +33,8 @@ class Service(file: String = "app.yml") {
val settings = SettingsLoader(
listOf(
Constant.recordSetIdPropertyName,
Constant.sftpBasePathPropertyName
Constant.sftpBasePathPropertyName,
Constant.hasThumbnailsPropertyName
),
file,
useStreamsConfig = true,
......
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
object NS {
val rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
val rdfs = "http://www.w3.org/2000/01/rdf-schema#"
val owl = "http://www.w3.org/2002/07/owl#"
val skos = "http://www.w3.org/2004/02/skos/core#"
val rico = "https://www.ica.org/standards/RiC/ontology#"
val ebucore = "http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#"
val memint = "https://memobase.ch/institution/"
val memrs = "https://memobase.ch/recordSet/"
val memr = "https://memobase.ch/record/"
val mempo = "https://memobase.ch/instantiation/physical/"
val memdo = "https://memobase.ch/instantiation/digital/"
val dce = "http://purl.org/dc/elements/1.1/"
val dct = "http://purl.org/dc/terms/"
val schema = "http://schema.org/"
val foaf = "http://xmlns.com/foaf/0.1/"
val wdt = "http://www.wikidata.org/prop/direct/"
val wdtn = "http://www.wikidata.org/prop/direct-normalized/"
val rdau = "http://rdaregistry.info/Elements/u/"
}
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
object RDF {
val type: Property = prop("type")
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.rdf, name)
}
private fun res(name: String): Resource {
return ResourceFactory.createResource(NS.rdf + name)
}
}
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.rdf
import org.apache.jena.rdf.model.Property
import org.apache.jena.rdf.model.Resource
import org.apache.jena.rdf.model.ResourceFactory
object RICO {
// Classes
val Record = res("Record")
val Instantiation = res("Instantiation")
val Title = res("Title")
val Language = res("Language")
val Identifier = res("Identifier")
val Place = res("Place")
val Agent = res("Agent")
val Person = res("Person")
val CorporateBody = res("CorporateBody")
val Rule = res("Rule")
val CreationRelation = res("CreationRelation")
val SingleDate = res("SingleDate")
val DateRange = res("DateRange")
val DateSet = res("DateSet")
// datatype properties
val title: Property = prop("title")
val source: Property = prop("source")
val descriptiveNote: Property = prop("descriptiveNote")
val name: Property = prop("name")
val type: Property = prop("type")
val expressedDate: Property = prop("expressedDate")
val normalizedDateValue: Property = prop("normalizedDateValue")
val normalizedValue: Property = prop("normalizedValue")
val scopeAndContent: Property = prop("scopeAndContent")
val physicalCharacteristics: Property = prop("physicalCharacteristics")
// Object Properties
val hasTitle: Property = prop("hasTitle")
val isTitleOf: Property = prop("isTitleOf")
val hasLanguage: Property = prop("hasLanguage")
val isLanguageOf: Property = prop("isLanguageOf")
val identifiedBy: Property = prop("identifiedBy")
val identifies: Property = prop("identifies")
val identifier: Property = prop("identifier")
val hasSubject: Property = prop("hasSubject")
val publishedBy: Property = prop("publishedBy")
val heldBy: Property = prop("heldBy")
val isPartOf: Property = prop("isPartOf")
val hasProvenance: Property = prop("hasProvenance")
val instantiates: Property = prop("instantiates")
val hasInstantiation: Property = prop("hasInstantiation")
val hasDerivedInstantiation: Property = prop("hasDerivedInstantiation")
val isDerivedFromInstantiation: Property = prop("isDerivedFromInstantiation")
val creationRelationHasSource: Property = prop("creationRelationHasSource")
val creationRelationHasTarget: Property = prop("creationRelationHasTarget")
val recordResourceOrInstantiationIsSourceOfCreationRelation: Property = prop("recordResourceOrInstantiationIsSourceOfCreationRelation")
val regulates: Property = prop("regulates")
val intellectualPropertyRightsHeldBy: Property = prop("intellectualPropertyRightsHeldBy")
val regulatedBy: Property = prop("regulatedBy")
private fun prop(name: String): Property {
return ResourceFactory.createProperty(NS.rico, name)
}
private fun res(name: String): Resource {
return ResourceFactory.createResource(NS.rico + name)
}
}
......@@ -5,6 +5,7 @@ sftp:
password: ${SFTP_PASSWORD:?system}
app:
recordSetId: ${RECORD_SET_ID:?system}
hasThumbnails: ${HAS_THUMBNAILS:?system}
sftp:
basePath: ${SFTP_BASE_PATH:?system}
kafka:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment