Commit f1eb8c76 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Add excel test

parent 0e0784e7
......@@ -24,5 +24,6 @@ data class TestParams(
val inputFileName: String,
val inputKey: String,
val expectedOutputKey: String,
val expectedOutputDirectoryName: String
val expectedOutputDirectoryName: String,
val processReportOutput: String
)
......@@ -32,7 +32,6 @@ import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.memobase.extensions.EmbeddedSftpServer
import org.memobase.settings.SettingsLoader
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class Tests {
......@@ -48,44 +47,44 @@ class Tests {
init {
sftpServer.createDirectories(
"/memobase/test_institution_1/test_record_set_1",
"/memobase/test_institution_2/test_record_set_2"
"/memobase/test_institution_2/test_record_set_2",
"/memobase/test_institution_3/test_record_set_3"
)
sftpServer.putFile(
"/memobase/test_institution_1/test_record_set_1/brandt_metadaten.csv",
FileInputStream("src/test/resources/sftp/brandt_metadaten.csv")
)
sftpServer.putFile(
"/memobase/test_institution_2/test_record_set_2/bauGAZH_metadaten.csv",
FileInputStream("src/test/resources/sftp/mapping_baugazh.csv")
)
sftpServer.putFile(
"/memobase/test_institution_3/test_record_set_3/bauGAZH_metadaten.csv",
FileInputStream("src/test/resources/sftp/excel_test_file.xlsx")
)
sftpServer.putFile("/memobase/test_institution_1/test_record_set_1/brandt_metadaten.csv", FileInputStream("src/test/resources/sftp/brandt_metadaten.csv"))
sftpServer.putFile("/memobase/test_institution_2/test_record_set_2/bauGAZH_metadaten.csv", FileInputStream("src/test/resources/sftp/mapping_baugazh.csv"))
}
@ParameterizedTest
@MethodSource("testParams")
fun `test inputs`(params: TestParams) {
val settingsLoader = SettingsLoader(
listOf(
"sheet",
"header.count",
"header.line",
"identifier"
),
params.settingsFileName,
useStreamsConfig = true,
readSftpSettings = true
)
val testDriver = TopologyTestDriver(KafkaTopology(settingsLoader).build(), settingsLoader.kafkaStreamsSettings)
val service = Service(params.settingsFileName)
val testDriver = TopologyTestDriver(service.topology, service.settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
)
testDriver.pipeInput(
factory.create(
settingsLoader.inputTopic, params.inputKey, readFile(params.inputFileName)
service.settings.inputTopic, params.inputKey, readFile(params.inputFileName)
)
)
var record = testDriver.readOutput(
settingsLoader.outputTopic,
service.settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
assertThat(record).isNotNull
var count = 0
while (record != null) {
count += 1
assertThat(record)
......@@ -94,9 +93,10 @@ class Tests {
.hasFieldOrPropertyWithValue("value", readFile(params.expectedOutputDirectoryName + "/$count.json"))
val reportedRecord = testDriver.readOutput(
"${settingsLoader.outputTopic}-reporting",
"${service.settings.outputTopic}-reporting",
StringDeserializer(),
StringDeserializer())
StringDeserializer()
)
if (reportedRecord != null) {
val data = reportedRecord.value()
......@@ -109,11 +109,21 @@ class Tests {
}
record = testDriver.readOutput(
settingsLoader.outputTopic,
service.settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
}
val processReport = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
assertThat(processReport)
.isNotNull
.hasFieldOrPropertyWithValue("value", params.processReportOutput)
}
private fun testParams() = Stream.of(
......@@ -123,7 +133,8 @@ class Tests {
"error_filter_input.json",
"brandt_metadaten.csv",
"brandt_metadaten.csv",
"error_filter_output"
"error_filter_output",
Klaxon().toJsonString(Report("brandt_metadaten.csv", "FAILURE", "No valid file found!"))
),
TestParams(
"valid csv input",
......@@ -131,7 +142,17 @@ class Tests {
"brandt_csv_import.json",
"brandt_metadaten.csv",
"AVGR13716",
"brandt_output"
"brandt_output",
Klaxon().toJsonString(Report("brandt_metadaten.csv", "SUCCESS", "Transformed table data into 1 records."))
),
TestParams(
"valid xlsx input",
"test3.yml",
"excel_test_input.json",
"excel_test_file.xlsx",
"AVGR13716",
"excel_output",
Klaxon().toJsonString(Report("", "FAILURE", "message"))
)
/*,
TestParams(
......
{"Exemplar-AVGRNr":"AVGR13716","Permalink":"https://www.gr.ch/Exemplare/13716","Titel-Title":"Wintersport in Arosa","Titel-Beschreibung":"Pferdesport; Ski alpin; Skispringen; Eishochey; Tourismus","Titel-ProduktionsjahrdesOriginals":"1920, 1920-1929, genaues Datum nicht eruierbar","Titel-FilmPersonen":"Brandt, Carl","Titel-Funktionen":"Autor/in","Titel-Genre":"Dokumentarfilm; Amateurfilm","Titel-Drehort":"Arosa","Titel-Weiteres":"Eisfest: teilweise identische Aufnahmen in AVGR12097 \u201eAnkunft David Zogg\u201c ; Schanzenspringen auf Carmenna: teilweise identische Aufnahmen in AVGR12115 \u201eTouristen auf dem Tschuggen\u201c","Titel-Stream-Url":"https://s3-eu-west-1.amazonaws.com/streaming.av-portal.gr.ch/13716/AVGR13716.mov","Titel-Benutzerzugang":"Intranet","Medium-Materialbezeichnung":"Film","Medium-MedienFormat":"35-mm-Film, Negativ und Positiv, Nitrat","Medium-Ton":"stumm","Medium-Farbe":"s/w getönt","Medium-Dauer":"0:17:02","Medium-Bandlaenge":"Vorhandene Elemente: AVGR9942: Negativ, Nitrat (CS, Z 986-172.8); AVGR9943: Positiv Nitrat (CS, Z 986-172.7); AVGR12098: Interpositiv / Marron 2366, Kopie 2016 (KBG); AVGR13715: Internegativ 2234, Kopie 2016 (KBG); AVGR13716: Positivkopie Farbe 2383, Kopie 2016, eingefärbte Sequenzen (KBG)"}
\ No newline at end of file
{"id" : "brandt_metadaten.csv", "message" : "Ignored message due to previous error.", "status" : "FAILURE"}
\ No newline at end of file
{
"path": "/memobase/test_institution_3/test_record_set_3/excel_test_file.xlsx",
"format": "XLSX"
}
\ No newline at end of file
id: jobXYZ
sftp:
host: localhost
port: 22000
user: user
password: password
app:
sheet: 1
header:
count: 3
line: 3
identifier: 1
kafka:
streams:
bootstrap.servers: localhost:12345
application.id: test-clinet-1234
topic:
in: test-topic-in
out: test-topic-out
process: process-topic-id-reporting
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment