Add excel test

src/test/kotlin/TestParams.kt

@@ -24,5 +24,6 @@ data class TestParams(
     val inputFileName: String,
     val inputKey: String,
     val expectedOutputKey: String,
-    val expectedOutputDirectoryName: String
+    val expectedOutputDirectoryName: String,
+    val processReportOutput: String
 )

src/test/kotlin/Tests.kt

@@ -32,7 +32,6 @@ import org.junit.jupiter.api.TestInstance
 import org.junit.jupiter.params.ParameterizedTest
 import org.junit.jupiter.params.provider.MethodSource
 import org.memobase.extensions.EmbeddedSftpServer
-import org.memobase.settings.SettingsLoader
 
 @TestInstance(TestInstance.Lifecycle.PER_CLASS)
 class Tests {
@@ -48,44 +47,44 @@ class Tests {
     init {
         sftpServer.createDirectories(
             "/memobase/test_institution_1/test_record_set_1",
-            "/memobase/test_institution_2/test_record_set_2"
+            "/memobase/test_institution_2/test_record_set_2",
+            "/memobase/test_institution_3/test_record_set_3"
+        )
+        sftpServer.putFile(
+            "/memobase/test_institution_1/test_record_set_1/brandt_metadaten.csv",
+            FileInputStream("src/test/resources/sftp/brandt_metadaten.csv")
+        )
+        sftpServer.putFile(
+            "/memobase/test_institution_2/test_record_set_2/bauGAZH_metadaten.csv",
+            FileInputStream("src/test/resources/sftp/mapping_baugazh.csv")
+        )
+        sftpServer.putFile(
+            "/memobase/test_institution_3/test_record_set_3/bauGAZH_metadaten.csv",
+            FileInputStream("src/test/resources/sftp/excel_test_file.xlsx")
         )
-        sftpServer.putFile("/memobase/test_institution_1/test_record_set_1/brandt_metadaten.csv", FileInputStream("src/test/resources/sftp/brandt_metadaten.csv"))
-        sftpServer.putFile("/memobase/test_institution_2/test_record_set_2/bauGAZH_metadaten.csv", FileInputStream("src/test/resources/sftp/mapping_baugazh.csv"))
     }
 
     @ParameterizedTest
     @MethodSource("testParams")
     fun `test inputs`(params: TestParams) {
-        val settingsLoader = SettingsLoader(
-            listOf(
-                "sheet",
-                "header.count",
-                "header.line",
-                "identifier"
-            ),
-            params.settingsFileName,
-            useStreamsConfig = true,
-            readSftpSettings = true
-        )
-        val testDriver = TopologyTestDriver(KafkaTopology(settingsLoader).build(), settingsLoader.kafkaStreamsSettings)
+        val service = Service(params.settingsFileName)
+        val testDriver = TopologyTestDriver(service.topology, service.settings.kafkaStreamsSettings)
         val factory = ConsumerRecordFactory(
             StringSerializer(), StringSerializer()
         )
         testDriver.pipeInput(
             factory.create(
-                settingsLoader.inputTopic, params.inputKey, readFile(params.inputFileName)
+                service.settings.inputTopic, params.inputKey, readFile(params.inputFileName)
             )
         )
 
         var record = testDriver.readOutput(
-            settingsLoader.outputTopic,
+            service.settings.outputTopic,
             StringDeserializer(),
             StringDeserializer()
         )
         assertThat(record).isNotNull
         var count = 0
-
         while (record != null) {
             count += 1
             assertThat(record)
@@ -94,9 +93,10 @@ class Tests {
                 .hasFieldOrPropertyWithValue("value", readFile(params.expectedOutputDirectoryName + "/$count.json"))
 
             val reportedRecord = testDriver.readOutput(
-                "${settingsLoader.outputTopic}-reporting",
+                "${service.settings.outputTopic}-reporting",
                 StringDeserializer(),
-                StringDeserializer())
+                StringDeserializer()
+            )
 
             if (reportedRecord != null) {
                 val data = reportedRecord.value()
@@ -109,11 +109,21 @@ class Tests {
             }
 
             record = testDriver.readOutput(
-                settingsLoader.outputTopic,
+                service.settings.outputTopic,
                 StringDeserializer(),
                 StringDeserializer()
             )
         }
+
+        val processReport = testDriver.readOutput(
+            service.settings.processReportTopic,
+            StringDeserializer(),
+            StringDeserializer()
+        )
+
+        assertThat(processReport)
+            .isNotNull
+            .hasFieldOrPropertyWithValue("value", params.processReportOutput)
     }
 
     private fun testParams() = Stream.of(
@@ -123,7 +133,8 @@ class Tests {
             "error_filter_input.json",
             "brandt_metadaten.csv",
             "brandt_metadaten.csv",
-            "error_filter_output"
+            "error_filter_output",
+            Klaxon().toJsonString(Report("brandt_metadaten.csv", "FAILURE", "No valid file found!"))
         ),
         TestParams(
             "valid csv input",
@@ -131,7 +142,17 @@ class Tests {
             "brandt_csv_import.json",
             "brandt_metadaten.csv",
             "AVGR13716",
-            "brandt_output"
+            "brandt_output",
+            Klaxon().toJsonString(Report("brandt_metadaten.csv", "SUCCESS", "Transformed table data into 1 records."))
+        ),
+        TestParams(
+            "valid xlsx input",
+            "test3.yml",
+            "excel_test_input.json",
+            "excel_test_file.xlsx",
+            "AVGR13716",
+            "excel_output",
+            Klaxon().toJsonString(Report("", "FAILURE", "message"))
         )
         /*,
         TestParams(

src/test/resources/data/excel_output/1.json

+{"Exemplar-AVGRNr":"AVGR13716","Permalink":"https://www.gr.ch/Exemplare/13716","Titel-Title":"Wintersport in Arosa","Titel-Beschreibung":"Pferdesport; Ski alpin; Skispringen; Eishochey;  Tourismus","Titel-ProduktionsjahrdesOriginals":"1920, 1920-1929, genaues Datum nicht eruierbar","Titel-FilmPersonen":"Brandt, Carl","Titel-Funktionen":"Autor/in","Titel-Genre":"Dokumentarfilm; Amateurfilm","Titel-Drehort":"Arosa","Titel-Weiteres":"Eisfest: teilweise identische Aufnahmen in AVGR12097 \u201eAnkunft David Zogg\u201c ; Schanzenspringen auf Carmenna: teilweise identische Aufnahmen in AVGR12115 \u201eTouristen auf dem Tschuggen\u201c","Titel-Stream-Url":"https://s3-eu-west-1.amazonaws.com/streaming.av-portal.gr.ch/13716/AVGR13716.mov","Titel-Benutzerzugang":"Intranet","Medium-Materialbezeichnung":"Film","Medium-MedienFormat":"35-mm-Film, Negativ und Positiv, Nitrat","Medium-Ton":"stumm","Medium-Farbe":"s/w getönt","Medium-Dauer":"0:17:02","Medium-Bandlaenge":"Vorhandene Elemente: AVGR9942: Negativ, Nitrat (CS, Z 986-172.8); AVGR9943: Positiv Nitrat (CS, Z 986-172.7); AVGR12098: Interpositiv / Marron 2366, Kopie 2016 (KBG); AVGR13715: Internegativ 2234, Kopie 2016 (KBG); AVGR13716: Positivkopie Farbe 2383, Kopie 2016, eingefärbte Sequenzen (KBG)"}
\ No newline at end of file

src/test/resources/data/excel_output/r1.json

+{"id" : "brandt_metadaten.csv", "message" : "Ignored message due to previous error.", "status" : "FAILURE"}
\ No newline at end of file

src/test/resources/data/excel_test_input.json

+{
+  "path": "/memobase/test_institution_3/test_record_set_3/excel_test_file.xlsx",
+  "format": "XLSX"
+}
\ No newline at end of file

src/test/resources/test3.yml

+id: jobXYZ
+sftp:
+  host: localhost
+  port: 22000
+  user: user
+  password: password
+app:
+  sheet: 1
+  header:
+    count: 3
+    line: 3
+  identifier: 1
+kafka:
+  streams:
+    bootstrap.servers: localhost:12345
+    application.id: test-clinet-1234
+  topic:
+    in: test-topic-in
+    out: test-topic-out
+    process: process-topic-id-reporting
\ No newline at end of file