Commit ce415629 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Allow for env specific subfolders.

parent 44bff976
Pipeline #27858 passed with stages
in 3 minutes and 57 seconds
......@@ -4,6 +4,7 @@ metadata:
name: "{{ .Values.shortSessionId }}-config"
namespace: memobase
data:
ENVIRONMENT: "{{ .Values.env }}"
SESSION_ID: "{{ .Values.sessionId }}"
RECORD_SET_ID: "{{ .Values.recordSetId }}"
INSTITUTION_ID: "{{ .Values.institutionId }}"
......
......@@ -22,6 +22,7 @@ recordSetId: placeholder
isPublished: false
drupalJobUuid: placeholder
drupalJobLogResultUuid: placeholder
env: placeholder
# Step 2
......
......@@ -18,15 +18,15 @@
package org.memobase
import java.io.File
import kotlin.system.exitProcess
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import ch.memobase.exceptions.SftpClientException
import ch.memobase.settings.SettingsLoader
import ch.memobase.sftp.SftpClient
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.SettingsLoader
import ch.memobase.sftp.SftpClient
import org.apache.logging.log4j.LogManager
import org.apache.logging.log4j.Logger
import java.io.File
import kotlin.system.exitProcess
class Service(fileName: String = "app.yml") {
private val log: Logger = LogManager.getLogger("TextFileValidationService")
......@@ -38,6 +38,7 @@ class Service(fileName: String = "app.yml") {
val settings =
SettingsLoader(
listOf(
"env",
"sessionId",
"recordSetId",
"institutionId",
......@@ -56,8 +57,21 @@ class Service(fileName: String = "app.yml") {
readSftpSettings = true
)
private val producer = Producer(settings.kafkaProducerSettings, settings.appSettings, settings.outputTopic, settings.processReportTopic)
private val producer = Producer(
settings.kafkaProducerSettings,
settings.appSettings,
settings.outputTopic,
settings.processReportTopic
)
private val recordSetId = settings.appSettings.getProperty("recordSetId")
private val basePath = run {
val env = settings.appSettings.getProperty("env")
if (env == "prod") {
recordSetId
} else {
"$env/$recordSetId"
}
}
private val sessionId = settings.appSettings.getProperty("sessionId")
private val validator = FileValidation()
......@@ -70,12 +84,12 @@ class Service(fileName: String = "app.yml") {
sftpClient.use { sftp ->
log.info("Connected to SFTP & Kafka.")
val files = try {
val fileList = sftp.listFiles(recordSetId).map { File(it) }
val fileList = sftp.listFiles(basePath).map { File(it) }
totalCount = fileList.size
fileList
} catch (ex: SftpClientException) {
ex.printStackTrace()
log.error("SFTP Exception: Could not compile the file list on sftp server in directory: './$recordSetId/'.")
log.error("SFTP Exception: Could not compile the file list on sftp server in directory: '$basePath'.")
val report = Report(
"$recordSetId#$sessionId",
status = ReportStatus.fatal,
......@@ -85,7 +99,7 @@ class Service(fileName: String = "app.yml") {
producer.sendReport(report)
exitProcess(1)
}
log.info("Retrieved file list from sftp server from folder: $recordSetId")
log.info("Retrieved file list from sftp server from folder: $basePath")
var reportCount = 0
var sftpClientExceptions = 0
try {
......
......@@ -4,6 +4,7 @@ sftp:
user: ${SFTP_USER:?env}
password: ${SFTP_PASSWORD:?env}
app:
env: ${ENVIRONMENT:?env}
sessionId: ${SESSION_ID:?env}
recordSetId: ${RECORD_SET_ID:?env}
institutionId: ${INSTITUTION_ID:?env}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment