Commit cc01a4ba authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor step name to be a config instead of a variable.

parent 8337414a
Pipeline #31044 passed with stages
in 4 minutes and 31 seconds
......@@ -17,4 +17,6 @@ recordSetIndexSource: prod-record-sets-index
kafkaConfigs: prod-kafka-bootstrap-servers
elasticConfigs: prod-elastic-configs
reportingStepName: 09.03-record-es-transform
env: prod
\ No newline at end of file
......@@ -17,4 +17,6 @@ recordSetIndexSource: stage-record-sets-index
kafkaConfigs: prod-kafka-bootstrap-servers
elasticConfigs: prod-elastic-configs
reportingStepName: 09.03-record-es-transform
env: stage
\ No newline at end of file
......@@ -17,4 +17,6 @@ recordSetIndexSource: test-record-sets-index
kafkaConfigs: test-kafka-bootstrap-servers
elasticConfigs: test-elastic-configs
reportingStepName: 09.03-record-es-transform
env: test
......@@ -18,4 +18,6 @@ recordSetIndexSource: prod-record-sets-index
kafkaConfigs: prod-kafka-bootstrap-servers
elasticConfigs: prod-elastic-configs
reportingStepName: 03-gi-institution-es-transform
env: prod
\ No newline at end of file
......@@ -18,4 +18,6 @@ recordSetIndexSource: stage-record-sets-index
kafkaConfigs: prod-kafka-bootstrap-servers
elasticConfigs: prod-elastic-configs
reportingStepName: 03-gi-institution-es-transform
env: stage
\ No newline at end of file
......@@ -18,4 +18,6 @@ recordSetIndexSource: test-record-sets-index
kafkaConfigs: test-kafka-bootstrap-servers
elasticConfigs: test-elastic-configs
reportingStepName: 03-gi-institution-es-transform
env: test
......@@ -18,4 +18,6 @@ recordSetIndexSource: prod-record-sets-index
kafkaConfigs: prod-kafka-bootstrap-servers
elasticConfigs: prod-elastic-configs
reportingStepName: 03-gi-record-set-es-transform
env: prod
\ No newline at end of file
......@@ -18,4 +18,6 @@ recordSetIndexSource: stage-record-sets-index
kafkaConfigs: prod-kafka-bootstrap-servers
elasticConfigs: prod-elastic-configs
reportingStepName: 03-gi-record-set-es-transform
env: stage
\ No newline at end of file
......@@ -18,4 +18,6 @@ recordSetIndexSource: test-record-sets-index
kafkaConfigs: test-kafka-bootstrap-servers
elasticConfigs: test-elastic-configs
reportingStepName: 03-gi-record-set-es-transform
env: test
......@@ -9,4 +9,5 @@ data:
UPDATE_TOPIC: "{{ .Values.updateTopic }}"
TOPIC_IN: "{{ .Values.inputTopic }}"
TOPIC_OUT: "{{ .Values.outputTopic }}"
TOPIC_PROCESS: "{{ .Values.reportingTopic }}"
\ No newline at end of file
TOPIC_PROCESS: "{{ .Values.reportingTopic }}"
REPORTING_STEP_NAME: "{{ .Values.reportingStepName }}"
\ No newline at end of file
......@@ -22,4 +22,6 @@ reuseStatementLabels: paceholder
mediaServerUrl: paceholder
updateTopic: placeholder
reportingStepName: placeholder
env: placeholder
\ No newline at end of file
......@@ -38,7 +38,8 @@ class App {
SettingsProps.documentsIndex,
SettingsProps.institutionIndex,
SettingsProps.recordSetIndex,
SettingsProps.updateTopic
SettingsProps.updateTopic,
SettingsProps.reportingStepName
),
file,
useStreamsConfig = true
......
......@@ -18,6 +18,7 @@
package ch.memobase
import ch.memobase.Service.Companion
import ch.memobase.helpers.Constants.SettingsProps
import ch.memobase.helpers.ElasticSearchWrapper
import ch.memobase.helpers.JsonUtility
......@@ -49,6 +50,7 @@ class KafkaTopology(
private val log = LogManager.getLogger("SearchDocService")
private val appSettings = settings.appSettings
private val step = appSettings.getProperty(SettingsProps.reportingStepName)
private val mediaUrl = appSettings.getProperty(SettingsProps.mediaUrl)
private val updateTopic = appSettings.getProperty(SettingsProps.updateTopic)
private val reportTopic = settings.processReportTopic
......@@ -85,13 +87,13 @@ class KafkaTopology(
readOnlyKey,
ReportStatus.success,
"",
Service.nameRecords
step
)
)
} catch (ex: InvalidInputException) {
Pair(
DocumentsSearchDoc.DEFAULT,
Report(readOnlyKey, ReportStatus.warning, ex.localizedMessage, Service.nameRecords)
Report(readOnlyKey, ReportStatus.warning, ex.localizedMessage, step)
)
}
}
......@@ -107,7 +109,7 @@ class KafkaTopology(
readOnlyKey,
ReportStatus.success,
"",
Service.nameInstitutions
step
)
)
} catch (ex: InvalidInputException) {
......@@ -116,7 +118,7 @@ class KafkaTopology(
readOnlyKey,
ReportStatus.warning,
ex.localizedMessage,
Service.nameInstitutions
step
)
)
}
......@@ -132,7 +134,7 @@ class KafkaTopology(
readOnlyKey,
ReportStatus.success,
"",
Service.nameRecordSets
step
)
)
} catch (ex: InvalidInputException) {
......@@ -141,7 +143,7 @@ class KafkaTopology(
readOnlyKey,
ReportStatus.warning,
ex.localizedMessage,
Service.nameRecordSets
step
)
)
}
......@@ -154,7 +156,7 @@ class KafkaTopology(
readOnlyKey,
ReportStatus.fatal,
"No record, memobase institution or record set present in input data: $value.",
Service.nameRecords
step
)
}
.to(reportTopic)
......
......@@ -37,9 +37,6 @@ import ch.memobase.helpers.TranslationMappers
class Service(settings: SettingsLoader) {
companion object {
const val nameRecords = "di-record-es-transform"
const val nameRecordSets = "gi-record-set-es-transform"
const val nameInstitutions = "gi-institution-es-transform"
}
private val log = LogManager.getLogger("SearchDocService")
......
......@@ -35,6 +35,7 @@ object Constants {
const val documentsIndex = "elastic.documentsIndex"
const val institutionIndex = "elastic.institutionIndex"
const val recordSetIndex = "elastic.recordSetIndex"
const val reportingStepName = "reportingStepName"
}
......
......@@ -12,6 +12,7 @@ app:
accessTermLabelsPath: "/configs/access_terms/labels.csv"
reuseStatementLabelsPath: "/configs/reuse_statements/labels.csv"
updateTopic: ${UPDATE_TOPIC:?system}
reportingStepName: ${REPORTING_STEP_NAME:?system}
kafka:
streams:
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:?system}
......
......@@ -69,7 +69,6 @@ class IntegrationTest {
)
)
val topology =
KafkaTopology(settings, TestUtilities.translationMappers, elasticSearchWrapperMocked)
val testDriver = TopologyTestDriver(topology.build(), settings.kafkaStreamsSettings)
......
......@@ -152,7 +152,7 @@ class TestInstitutionSearchDoc {
"completeExampleTest",
ReportStatus.success,
"",
Service.nameInstitutions
"test"
)
)
}
......
......@@ -231,7 +231,7 @@ class TestRecordSetSearchDoc {
"testComplete",
ReportStatus.success,
"",
Service.nameRecordSets
"test"
)
)
}
......
......@@ -12,6 +12,7 @@ app:
accessTermLabelsPath: "src/test/resources/configs/access-term-labels.csv"
reuseStatementLabelsPath: "src/test/resources/configs/reuse-statement-labels.csv"
updateTopic: "documents-v18"
reportingStepName: "test"
kafka:
streams:
bootstrap.servers: localhost:12345
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment