Update dependency & implement shutdown hook

build.gradle

@@ -32,9 +32,6 @@ ext {
 }
 
 dependencies {
-    // https://mvnrepository.com/artifact/org.elasticsearch.client/elasticsearch-rest-high-level-client
-    //compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.1.0'
-
     // Logging Framework
     implementation "org.apache.logging.log4j:log4j-api:${log4jV}"
     implementation "org.apache.logging.log4j:log4j-core:${log4jV}"
@@ -44,7 +41,8 @@ dependencies {
     implementation group: 'org.apache.kafka', name: 'kafka-clients', version: kafkaV
     implementation "org.apache.kafka:kafka-streams:${kafkaV}"
 
-    implementation 'org.memobase:memobase-service-utilities:1.4.0'
+    implementation 'org.memobase:memobase-service-utilities:1.12.1'
+    implementation 'ch.memobase:import-process-effects-registry_2.12:0.2.1'
     // CSV Reader
     implementation("com.github.doyaaaaaken:kotlin-csv-jvm:0.7.3")
     // XSLX / XSL Reader
@@ -55,8 +53,6 @@ dependencies {
 
     // JSON Parser
     implementation 'com.beust:klaxon:5.2'
-    // Compression
-    //implementation "org.apache.commons:commons-compress:1.19"
 
     implementation 'org.jetbrains.kotlin:kotlin-stdlib-jdk8'
     implementation "org.jetbrains.kotlin:kotlin-script-runtime:1.3.71"

src/main/kotlin/Extensions.kt

+package org.memobase
+
+import org.apache.poi.ss.usermodel.CellType
+import org.apache.poi.ss.usermodel.Row
+import org.apache.poi.ss.usermodel.Sheet
+
+fun Sheet.filterEmptyRows(): Iterable<Row> {
+    return this.filter { row ->
+        row.any { cell ->
+            // filter all rows with no string, boolean or numeric cell
+            when (cell.cellType) {
+                CellType._NONE -> false
+                CellType.NUMERIC -> true
+                CellType.STRING -> true
+                CellType.FORMULA -> false
+                CellType.BLANK -> false
+                CellType.BOOLEAN -> true
+                CellType.ERROR -> false
+                else -> false
+            }
+        }
+    }
+}
\ No newline at end of file

src/main/kotlin/KafkaTopology.kt

@@ -41,7 +41,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
 
     private val klaxon = Klaxon()
 
-    fun build(): Topology {
+    fun prepare(): StreamsBuilder {
         val builder = StreamsBuilder()
 
         val branchedSource = builder
@@ -57,7 +57,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
         processValidInput(branchedSource[1]
                 .mapValues { key, value -> parser.parseTable(key, value) })
 
-        return builder.build()
+        return builder
     }
 
     private fun processValidInput(stream: KStream<String, ParserResult>) {

src/main/kotlin/Service.kt

@@ -18,9 +18,15 @@
 
 package org.memobase
 
+import ch.memobase.Effect
+import ch.memobase.EffectsRegistry
+import ch.memobase.ShutdownMessage
 import org.apache.kafka.streams.KafkaStreams
 import org.apache.logging.log4j.LogManager
 import org.memobase.settings.SettingsLoader
+import scala.Some
+import scala.runtime.BoxedUnit
+import kotlin.system.exitProcess
 
 class Service(file: String = "app.yml") {
     private val log = LogManager.getLogger("TableDataService")
@@ -37,10 +43,16 @@ class Service(file: String = "app.yml") {
             readSftpSettings = true
     )
 
-    val topology = KafkaTopology(settings).build()
-    private val stream = KafkaStreams(topology, settings.kafkaStreamsSettings)
+    private val appId = settings.kafkaStreamsSettings.getProperty("application.id")
+    val builder = KafkaTopology(settings).prepare()
+
+    private val registry = EffectsRegistry()
+    private val shutdownEffect = Effect("shutdown", this::exit, Some("Shutting down application"))
 
     fun run() {
+        registry.register(ShutdownMessage(appId.replace("-normalization-service", ""), "normalization-service", "termination"), shutdownEffect)
+        registry.run(builder, "import-process-admin")
+        val stream = KafkaStreams(builder.build(), settings.kafkaStreamsSettings)
         stream.use {
             it.start()
             while (stream.state().isRunning) {
@@ -49,4 +61,8 @@ class Service(file: String = "app.yml") {
             }
         }
     }
+
+    private fun exit(): BoxedUnit {
+        exitProcess(0)
+    }
 }

src/main/kotlin/UtilityFunctions.kt

@@ -22,6 +22,7 @@ import org.apache.poi.ss.usermodel.Cell
 import org.apache.poi.ss.usermodel.CellType
 import org.apache.poi.ss.usermodel.Row
 
+
 object UtilityFunctions {
 
     /**

src/test/kotlin/TestIntegration.kt

@@ -74,7 +74,7 @@ class TestIntegration {
     @MethodSource("testParams")
     fun `test kafka integrations`(params: TestParams) {
         val service = Service(params.settingsFileName)
-        val testDriver = TopologyTestDriver(service.topology, service.settings.kafkaStreamsSettings)
+        val testDriver = TopologyTestDriver(service.builder.build(), service.settings.kafkaStreamsSettings)
         val factory = ConsumerRecordFactory(
             StringSerializer(), StringSerializer()
         )