Commit 9f55e533 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Fix base path issue.

parent 5a8e4ee4
Pipeline #10650 passed with stages
in 3 minutes and 59 seconds
......@@ -5,8 +5,8 @@ metadata:
namespace: memobase
data:
APPLICATION_ID: "{{ .Values.processId }}-{{ .Values.jobName }}"
INSTITUTION_ID: "{{ .Values.institutionId }}"
RECORD_SET_ID: "{{ .Values.recordSetId }}"
SFTP_BASE_PATH: "{{ .Values.sftpBasePath }}"
TOPIC_IN: "{{ .Values.processId }}-{{ .Values.lastJobName }}"
TOPIC_OUT: "{{ .Values.processId }}-{{ .Values.jobName }}"
TOPIC_PROCESS: "{{ .Values.processId }}-reporting"
\ No newline at end of file
......@@ -19,4 +19,5 @@ jobName: media-linker
processId: p0001
institutionId: placeholder
recordSetId: placeholder
\ No newline at end of file
recordSetId: placeholder
sftpBasePath: /swissbib_index/mb_sftp
\ No newline at end of file
......@@ -43,15 +43,18 @@ class KafkaTopology(private val settings: SettingsLoader) {
private val log = LogManager.getLogger("MediaLinker")
private val sftpClient = SftpClient(settings.sftpSettings)
private val sftpBasePath = settings.appSettings.getProperty("sftp.basePath")
private val files = createMediaFileList(settings.appSettings.getProperty("recordSetId"))
private fun createMediaFileList(setting: String): List<String> {
return try {
sftpClient.listFiles("/$setting/media")
val list = sftpClient.listFiles("$sftpBasePath/$setting/media")
log.info("Files found on sftp server: $list.")
list
} catch (ex: Exception) {
ex.printStackTrace()
log.error(ex.localizedMessage + ": /$setting/media")
log.error(ex.localizedMessage + ": $sftpBasePath/$setting/media")
exitProcess(1)
}
}
......
......@@ -22,15 +22,16 @@ import org.apache.logging.log4j.LogManager
import org.memobase.settings.SettingsLoader
class Service(file: String = "app.yml") {
private val log = LogManager.getLogger("TableDataService")
private val log = LogManager.getLogger("MediaLinkerService")
val settings = SettingsLoader(
listOf(
"recordSetId"
),
file,
useStreamsConfig = true,
readSftpSettings = true
listOf(
"recordSetId",
"sftp.basePath"
),
file,
useStreamsConfig = true,
readSftpSettings = true
)
val topology = KafkaTopology(settings).build()
......
......@@ -5,6 +5,8 @@ sftp:
password: ${SFTP_PASSWORD:?system}
app:
recordSetId: ${RECORD_SET_ID:?system}
sftp:
basePath: ${SFTP_BASE_PATH:?system}
kafka:
streams:
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:?system}
......
......@@ -53,9 +53,9 @@ class Tests {
init {
val files = listOf(
Pair("/test_record_set_1/media", "MEI_49884.jpg"),
Pair("/test_record_set_1/media", "MEI_49885.jpg"),
Pair("/test_record_set_1/media", "MEI_49886.jpg")
Pair("/base/test_record_set_1/media", "MEI_49884.jpg"),
Pair("/base/test_record_set_1/media", "MEI_49885.jpg"),
Pair("/base/test_record_set_1/media", "MEI_49886.jpg")
)
for (pair in files) {
......
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#locator> "sftp:/test_record_set_1/media/MEI_49885.jpg" .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <http://www.ebu.ch/metadata/ontologies/ebucore/ebucore#locator> "sftp:/base/test_record_set_1/media/MEI_49885.jpg" .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/instantiation/digital/BAZ-MEI_49885-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
......
......@@ -5,6 +5,8 @@ sftp:
password: password
app:
recordSetId: test_record_set_1
sftp:
basePath: /base
kafka:
streams:
bootstrap.servers: localhost:12345
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment