Update configs & helm chart

helm-charts/helm-values/di-mapper-prod.yml

+deploymentName: di-mapper-prod
+kafkaConfigs: prod-kafka-bootstrap-servers
+applicationId: di-mapper-prod-app
+inputTopicName: mb-di-mapper-prod
+outputTopicName: mb-di-media-linker-prod
+reportingTopicName: mb-di-reports-prod
+configTopicName: mb-di-config-prod
\ No newline at end of file

helm-charts/helm-values/di-mapper-stage.yml

+deploymentName: di-mapper-stage
+kafkaConfigs: prod-kafka-bootstrap-servers
+applicationId: di-mapper-stage-app
+inputTopicName: mb-di-mapper-stage
+outputTopicName: mb-di-media-linker-stage
+reportingTopicName: mb-di-reports-stage
+configTopicName: mb-di-config-stage
\ No newline at end of file

helm-charts/helm-values/di-mapper-test.yml

+deploymentName: di-mapper-test
+kafkaConfigs: test-kafka-bootstrap-servers
+applicationId: di-mapper-test-app
+inputTopicName: mb-di-mapper-test
+outputTopicName: mb-di-media-linker-test
+reportingTopicName: mb-di-reports-test
+configTopicName: mb-di-config-test
\ No newline at end of file

helm-charts/helm-values/ip-mapper-service.yaml

-kafkaConfigs: prod-kafka-bootstrap-servers
-deploymentName: ip-mapper-service
-applicationId: mapper-service-app
-inputTopicName: import-process-mapper
-outputTopicName: import-process-media-linker
-reportingTopicName: import-process-reporting
\ No newline at end of file

helm-charts/templates/app-config.yml

@@ -5,6 +5,7 @@ metadata:
   namespace: memobase
 data:
   APPLICATION_ID: {{ .Values.applicationId }}
+  SERVICE_CONFIG_TOPIC: {{ .Values.configTopicName }}
   TOPIC_IN: {{ .Values.inputTopicName }}
   TOPIC_OUT: {{ .Values.outputTopicName }}
   TOPIC_PROCESS: {{ .Values.reportingTopicName }}
\ No newline at end of file

helm-charts/values.yaml

@@ -2,9 +2,10 @@ registry: "cr.gitlab.switch.ch"
 image: "memoriav/memobase-2020/services/import-process/mapper-service"
 tag: "latest"
 
-kafkaConfigs: prod-kafka-bootstrap-servers
-deploymentName: mapper-service
-applicationId: mapper-service-app
-inputTopicName: import-process-mapper
-outputTopicName: import-process-media-linker
-reportingTopicName: import-process-reporting
\ No newline at end of file
+kafkaConfigs: placeholder
+deploymentName: placeholder
+applicationId: placeholder
+inputTopicName: placeholder
+outputTopicName: placeholder
+reportingTopicName: placeholder
+configTopicName: placeholder
\ No newline at end of file

src/main/kotlin/KafkaTopology.kt

@@ -66,7 +66,7 @@ class KafkaTopology(
     fun prepare(): StreamsBuilder {
         val builder = StreamsBuilder()
 
-        val configStream = builder.stream<String, String>("import-process-config")
+        val configStream = builder.stream<String, String>(settings.appSettings.getProperty(Service.configTopicNameProp))
             .map { key, value -> KeyValue(key.toByteArray(), value.toByteArray()) }
 
         val stream = builder.stream<String, String>(settings.inputTopic)

src/main/kotlin/Service.kt

@@ -25,12 +25,15 @@ import ch.memobase.settings.SettingsLoader
 class Service(file: String = "app.yml") {
     companion object {
         const val step = "mapper-service"
+        const val configTopicNameProp = "configTopic"
     }
 
     private val log = LogManager.getLogger("MapperService")
 
     val settings = SettingsLoader(
-        listOf(),
+        listOf(
+            configTopicNameProp
+        ),
         file,
         useStreamsConfig = true
     )

src/main/resources/app.yml

+app:
+  configTopic: ${SERVICE_CONFIG_TOPIC:?system}
 kafka:
   streams:
     bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:?system}

src/test/kotlin/IntegrationTests.kt

@@ -95,7 +95,7 @@ class IntegrationTests {
 
         testDriver.pipeInput(
             factory.create(
-                "import-process-config",
+                "mb-di-config-test",
                 "ati-002#mapping",
                 readFile("kafkaTests/${params.count}/config/mapping.yml")
             )

src/test/resources/kafkaTest1.yml

 app:
-  institutionId: "BAZ"
-  recordSetId: "BAZ-B_MEI"
-  configs: src/test/resources/kafkaTests/1/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345

src/test/resources/kafkaTest2.yml

 app:
-  institutionId: "TEST"
-  recordSetId: "TEST_RECORD_SET"
-  configs: src/test/resources/kafkaTests/2/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345

src/test/resources/kafkaTest3.yml

 app:
-  institutionId: "TEST"
-  recordSetId: "TEST_RECORD_SET"
-  configs: src/test/resources/kafkaTests/3/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345

src/test/resources/kafkaTest4.yml

 app:
-  institutionId: "TEST"
-  recordSetId: "TEST_RECORD_SET"
-  configs: src/test/resources/kafkaTests/4/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345

src/test/resources/kafkaTest5.yml

 app:
-  institutionId: "TEST"
-  recordSetId: "TEST_RECORD_SET"
-  configs: src/test/resources/kafkaTests/5/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345

src/test/resources/kafkaTest6.yml

 app:
-  institutionId: "TEST"
-  recordSetId: "TEST_RECORD_SET"
-  configs: src/test/resources/kafkaTests/6/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345

src/test/resources/kafkaTest7.yml

 app:
-  institutionId: "TEST"
-  recordSetId: "TEST_RECORD_SET"
-  configs: src/test/resources/kafkaTests/7/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345

src/test/resources/kafkaTest8.yml

 app:
-  institutionId: "TEST"
-  recordSetId: "TEST_RECORD_SET"
-  configs: src/test/resources/kafkaTests/8/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345

src/test/resources/kafkaTest9.yml

 app:
-  institutionId: "TEST"
-  recordSetId: "TEST_RECORD_SET"
-  configs: src/test/resources/kafkaTests/9/config
-  isPublished: false
+  configTopic: mb-di-config-test
 kafka:
   streams:
     bootstrap.servers: localhost:12345