[WIP] First changes to read thumbnails

build.gradle

@@ -47,7 +47,7 @@ dependencies {
     // JSON Parser
     implementation 'com.beust:klaxon:5.2'
 
-    implementation 'org.memobase:memobase-service-utilities:1.12.1'
+    implementation 'org.memobase:memobase-service-utilities:0.14.2'
     implementation 'ch.memobase:import-process-effects-registry_2.12:0.2.1'
 
     implementation 'org.apache.jena:apache-jena:3.14.0'

helm-charts/templates/app-config.yml

@@ -7,6 +7,7 @@ data:
   APPLICATION_ID: "{{ .Values.processId }}-{{ .Values.jobName }}"
   RECORD_SET_ID: "{{ .Values.recordSetId }}"
   SFTP_BASE_PATH: "{{ .Values.sftpBasePath }}"
+  HAS_THUMBNAILS: "{{ .Values.hasThumbnails }}"
   TOPIC_IN: "{{ .Values.processId }}-{{ .Values.lastJobName }}"
   TOPIC_OUT: "{{ .Values.processId }}-{{ .Values.jobName }}"
   TOPIC_PROCESS: "{{ .Values.processId }}-reporting"
\ No newline at end of file

helm-charts/values.yaml

@@ -20,4 +20,5 @@ processId: p0001
 
 institutionId: placeholder
 recordSetId: placeholder
-sftpBasePath: /swissbib_index/mb_sftp
\ No newline at end of file
+sftpBasePath: /swissbib_index/mb_sftp
+hasThumbnails: false
\ No newline at end of file

src/main/kotlin/Constant.kt

@@ -2,9 +2,11 @@ package org.memobase
 
 object Constant {
     const val mediaFolderName = "media"
+    const val thumbnailFolderName ="thumbnails"
     const val recordSetIdPropertyName = "recordSetId"
     const val sftpBasePathPropertyName = "sftp.basePath"
     const val topicReportingSuffix = "reporting"
+    const val hasThumbnailsPropertyName = "hasThumbnails"
 
     const val rdfParserLang = "NTRIPLES"
 

src/main/kotlin/FolderNotFoundException.kt

+package org.memobase
+
+class FolderNotFoundException(folder: String) : Exception("Folder $folder does not exist on sftp server.")
\ No newline at end of file

src/main/kotlin/KafkaTopology.kt

@@ -18,13 +18,9 @@
 
 package org.memobase
 
-import java.io.StringReader
-import java.io.StringWriter
-import kotlin.system.exitProcess
 import org.apache.jena.rdf.model.Model
 import org.apache.jena.rdf.model.ModelFactory
 import org.apache.jena.rdf.model.Resource
-import org.apache.jena.rdf.model.Statement
 import org.apache.jena.rdf.model.impl.StatementImpl
 import org.apache.kafka.streams.StreamsBuilder
 import org.apache.kafka.streams.kstream.Predicate
@@ -36,12 +32,23 @@ import org.memobase.reports.ReportMessages
 import org.memobase.reports.ReportStatus
 import org.memobase.settings.SettingsLoader
 import org.memobase.sftp.SftpClient
+import java.io.StringReader
+import java.io.StringWriter
+import kotlin.system.exitProcess
 
 class KafkaTopology(private val settings: SettingsLoader) {
     private val log = LogManager.getLogger("MediaLinker")
 
+    private val appSettings = settings.appSettings
+
+    private val fileFetcher = LocatorFetcher(
+        settings.sftpSettings,
+        appSettings.getProperty(Constant.sftpBasePathPropertyName),
+        appSettings.getProperty(Constant.recordSetIdPropertyName),
+        appSettings.getProperty(Constant.hasThumbnailsPropertyName)!!.toBoolean()
+    )
     private val sftpClient = SftpClient(settings.sftpSettings)
-    private val sftpBasePath = settings.appSettings.getProperty(Constant.sftpBasePathPropertyName)
+    private val sftpBasePath = appSettings.getProperty(Constant.sftpBasePathPropertyName)
 
     private val files = createMediaFileList(settings.appSettings.getProperty(Constant.recordSetIdPropertyName))
 

src/main/kotlin/rdf/EBUCORE.kt → src/main/kotlin/LocatorFetcher.kt

 /*
- * mapper-service
+ * Media Linker
  * Copyright (C) 2020 Memoriav
  *
  * This program is free software: you can redistribute it and/or modify
@@ -15,30 +15,43 @@
  * You should have received a copy of the GNU Affero General Public License
  * along with this program.  If not, see <https://www.gnu.org/licenses/>.
  */
+package org.memobase
 
-package org.memobase.rdf
+import org.apache.log4j.LogManager
+import org.memobase.settings.SftpSettings
+import org.memobase.sftp.SftpClient
 
-import org.apache.jena.rdf.model.Property
-import org.apache.jena.rdf.model.Resource
-import org.apache.jena.rdf.model.ResourceFactory
+class LocatorFetcher(sftpSettings: SftpSettings, basePath: String, path: String, private val hasThumbnails: Boolean) {
+    private val log = LogManager.getLogger("LocatorFetcher")
 
-object EBUCORE {
+    private val client = SftpClient(sftpSettings)
+    private val pathMedia = "$basePath/$path/${Constant.mediaFolderName}"
+    private val pathThumbnails = "$basePath/$path/${Constant.thumbnailFolderName}"
 
-    val hasGenre = prop("hasGenre")
-    val hasFormat = prop("hasFormat")
-    val hasMedium = prop("hasMedium")
-    val duration = prop("duration")
-    val displayAspectRatio = prop("displayAspectRatio")
-    val audioTrackConfiguration = prop("audioTrackConfiguration")
-    val playbackSpeed = prop("playbackSpeed")
-    val hasStandard = prop("hasStandard")
-    val locator = prop("locator")
+    fun thumbnailFiles(): List<String> {
+        return if (hasThumbnails) {
+            if (!client.exists(pathThumbnails)) {
+                throw FolderNotFoundException(pathThumbnails)
+            }
+            log.info("Found thumbnail folder on sftp server: $pathThumbnails.")
+            client.listFiles(pathThumbnails)
+        } else {
+            emptyList()
+        }
+    }
 
-    private fun prop(name: String): Property {
-        return ResourceFactory.createProperty(NS.ebucore, name)
+    fun mediaFiles(): List<String> {
+        return if (client.exists(pathMedia)) {
+            log.info("Found media folder on sftp server: $pathMedia")
+            client.listFiles(pathMedia)
+        } else {
+            throw FolderNotFoundException(pathMedia)
+        }
     }
 
-    private fun res(name: String): Resource {
-        return ResourceFactory.createResource(NS.ebucore + name)
+    fun fetchThumbnails() {
+
+
     }
-}
+
+}
\ No newline at end of file

src/main/kotlin/Service.kt

@@ -33,7 +33,8 @@ class Service(file: String = "app.yml") {
     val settings = SettingsLoader(
         listOf(
             Constant.recordSetIdPropertyName,
-            Constant.sftpBasePathPropertyName
+            Constant.sftpBasePathPropertyName,
+            Constant.hasThumbnailsPropertyName
         ),
         file,
         useStreamsConfig = true,

src/main/kotlin/rdf/NS.kt

-/*
- * mapper-service
- * Copyright (C) 2020 Memoriav
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.memobase.rdf
-
-object NS {
-    val rdf = "http://www.w3.org/1999/02/22-rdf-syntax-ns#"
-    val rdfs = "http://www.w3.org/2000/01/rdf-schema#"
-    val owl = "http://www.w3.org/2002/07/owl#"
-    val skos = "http://www.w3.org/2004/02/skos/core#"
-
-    val rico = "https://www.ica.org/standards/RiC/ontology#"
-    val ebucore = "http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#"
-
-    val memint = "https://memobase.ch/institution/"
-    val memrs = "https://memobase.ch/recordSet/"
-    val memr = "https://memobase.ch/record/"
-    val mempo = "https://memobase.ch/instantiation/physical/"
-    val memdo = "https://memobase.ch/instantiation/digital/"
-
-    val dce = "http://purl.org/dc/elements/1.1/"
-    val dct = "http://purl.org/dc/terms/"
-
-    val schema = "http://schema.org/"
-    val foaf = "http://xmlns.com/foaf/0.1/"
-    val wdt = "http://www.wikidata.org/prop/direct/"
-    val wdtn = "http://www.wikidata.org/prop/direct-normalized/"
-
-    val rdau = "http://rdaregistry.info/Elements/u/"
-}

src/main/kotlin/rdf/RDF.kt

-/*
- * mapper-service
- * Copyright (C) 2020 Memoriav
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.memobase.rdf
-
-import org.apache.jena.rdf.model.Property
-import org.apache.jena.rdf.model.Resource
-import org.apache.jena.rdf.model.ResourceFactory
-
-object RDF {
-
-    val type: Property = prop("type")
-
-    private fun prop(name: String): Property {
-        return ResourceFactory.createProperty(NS.rdf, name)
-    }
-
-    private fun res(name: String): Resource {
-        return ResourceFactory.createResource(NS.rdf + name)
-    }
-}

src/main/kotlin/rdf/RICO.kt

-/*
- * mapper-service
- * Copyright (C) 2020 Memoriav
- *
- * This program is free software: you can redistribute it and/or modify
- * it under the terms of the GNU Affero General Public License as published by
- * the Free Software Foundation, either version 3 of the License, or
- * (at your option) any later version.
- *
- * This program is distributed in the hope that it will be useful,
- * but WITHOUT ANY WARRANTY; without even the implied warranty of
- * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
- * GNU Affero General Public License for more details.
- *
- * You should have received a copy of the GNU Affero General Public License
- * along with this program.  If not, see <https://www.gnu.org/licenses/>.
- */
-
-package org.memobase.rdf
-
-import org.apache.jena.rdf.model.Property
-import org.apache.jena.rdf.model.Resource
-import org.apache.jena.rdf.model.ResourceFactory
-
-object RICO {
-
-    // Classes
-
-    val Record = res("Record")
-    val Instantiation = res("Instantiation")
-    val Title = res("Title")
-    val Language = res("Language")
-    val Identifier = res("Identifier")
-    val Place = res("Place")
-    val Agent = res("Agent")
-    val Person = res("Person")
-    val CorporateBody = res("CorporateBody")
-    val Rule = res("Rule")
-
-    val CreationRelation = res("CreationRelation")
-
-    val SingleDate = res("SingleDate")
-    val DateRange = res("DateRange")
-    val DateSet = res("DateSet")
-
-    // datatype properties
-
-    val title: Property = prop("title")
-    val source: Property = prop("source")
-    val descriptiveNote: Property = prop("descriptiveNote")
-    val name: Property = prop("name")
-    val type: Property = prop("type")
-    val expressedDate: Property = prop("expressedDate")
-    val normalizedDateValue: Property = prop("normalizedDateValue")
-    val normalizedValue: Property = prop("normalizedValue")
-    val scopeAndContent: Property = prop("scopeAndContent")
-    val physicalCharacteristics: Property = prop("physicalCharacteristics")
-
-    // Object Properties
-
-    val hasTitle: Property = prop("hasTitle")
-    val isTitleOf: Property = prop("isTitleOf")
-    val hasLanguage: Property = prop("hasLanguage")
-    val isLanguageOf: Property = prop("isLanguageOf")
-
-    val identifiedBy: Property = prop("identifiedBy")
-    val identifies: Property = prop("identifies")
-    val identifier: Property = prop("identifier")
-
-    val hasSubject: Property = prop("hasSubject")
-
-    val publishedBy: Property = prop("publishedBy")
-
-    val heldBy: Property = prop("heldBy")
-    val isPartOf: Property = prop("isPartOf")
-    val hasProvenance: Property = prop("hasProvenance")
-
-    val instantiates: Property = prop("instantiates")
-    val hasInstantiation: Property = prop("hasInstantiation")
-    val hasDerivedInstantiation: Property = prop("hasDerivedInstantiation")
-    val isDerivedFromInstantiation: Property = prop("isDerivedFromInstantiation")
-
-    val creationRelationHasSource: Property = prop("creationRelationHasSource")
-    val creationRelationHasTarget: Property = prop("creationRelationHasTarget")
-    val recordResourceOrInstantiationIsSourceOfCreationRelation: Property = prop("recordResourceOrInstantiationIsSourceOfCreationRelation")
-
-    val regulates: Property = prop("regulates")
-    val intellectualPropertyRightsHeldBy: Property = prop("intellectualPropertyRightsHeldBy")
-    val regulatedBy: Property = prop("regulatedBy")
-
-    private fun prop(name: String): Property {
-        return ResourceFactory.createProperty(NS.rico, name)
-    }
-
-    private fun res(name: String): Resource {
-        return ResourceFactory.createResource(NS.rico + name)
-    }
-}

src/main/resources/app.yml

@@ -5,6 +5,7 @@ sftp:
   password: ${SFTP_PASSWORD:?system}
 app:
   recordSetId: ${RECORD_SET_ID:?system}
+  hasThumbnails: ${HAS_THUMBNAILS:?system}
   sftp:
     basePath: ${SFTP_BASE_PATH:?system}
 kafka: