Commit 47dc3f5e authored by Jonas Waeber's avatar Jonas Waeber

Fix namespace removal test & code.

parent 3d32309a
Pipeline #20241 passed with stages
in 5 minutes and 15 seconds
...@@ -65,6 +65,7 @@ class KafkaTopology( ...@@ -65,6 +65,7 @@ class KafkaTopology(
.mapValues { value -> JSON.parse(value) } .mapValues { value -> JSON.parse(value) }
.filter { _, value -> value.isNotEmpty() } .filter { _, value -> value.isNotEmpty() }
.mapValues { value -> JSON.unpack(value) } .mapValues { value -> JSON.unpack(value) }
.map { key, value -> KeyValue(key.substringAfterLast("/"), value) }
.branch( .branch(
Predicate { _, value -> value.containsKey(JSON.recordTag) }, Predicate { _, value -> value.containsKey(JSON.recordTag) },
Predicate { _, value -> value.containsKey(JSON.institutionTag) }, Predicate { _, value -> value.containsKey(JSON.institutionTag) },
...@@ -155,7 +156,6 @@ class KafkaTopology( ...@@ -155,7 +156,6 @@ class KafkaTopology(
jsonWriter.writeValue(out, value) jsonWriter.writeValue(out, value)
out.toString() out.toString()
} }
.map { key, value -> KeyValue(key.substringAfterLast("/"), value) }
.to(settings.outputTopic) .to(settings.outputTopic)
} }
} }
...@@ -28,6 +28,7 @@ import org.apache.kafka.streams.test.ConsumerRecordFactory ...@@ -28,6 +28,7 @@ import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertAll
import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource import org.junit.jupiter.params.provider.MethodSource
import org.memobase.model.FacetContainer import org.memobase.model.FacetContainer
...@@ -49,9 +50,11 @@ class IntegrationTest { ...@@ -49,9 +50,11 @@ class IntegrationTest {
val settings = App.createSettings("kafkaTest1.yml") val settings = App.createSettings("kafkaTest1.yml")
every { TestUtilities.elasticSearchWrapperMocked.countNumberOfDocuments("") } returns 123 every { TestUtilities.elasticSearchWrapperMocked.countNumberOfDocuments("") } returns 123
every { TestUtilities.elasticSearchWrapperMocked.getDocumentTypesFromRecords("", "") } returns listOf(FacetContainer( every { TestUtilities.elasticSearchWrapperMocked.getDocumentTypesFromRecords("", "") } returns listOf(
LanguageContainer(listOf("Foto"), listOf("Foto"), listOf("Foto"), emptyList()), "Foto", emptyList() FacetContainer(
)) LanguageContainer(listOf("Foto"), listOf("Foto"), listOf("Foto"), emptyList()), "Foto", emptyList()
)
)
val topology = val topology =
...@@ -72,8 +75,13 @@ class IntegrationTest { ...@@ -72,8 +75,13 @@ class IntegrationTest {
StringDeserializer() StringDeserializer()
) )
assertThat(record.value()) assertAll("",
.isEqualTo(readFile("${params.count}/output.json")) {
assertThat(record.value())
.isEqualTo(readFile("${params.count}/output.json"))
},
{ assertThat(record.key()).isEqualTo(params.outputKey) }
)
} }
private fun testParams() = Stream.of( private fun testParams() = Stream.of(
...@@ -81,18 +89,21 @@ class IntegrationTest { ...@@ -81,18 +89,21 @@ class IntegrationTest {
"test-1", "test-1",
1, 1,
"kafkaTest1.yml", "kafkaTest1.yml",
"LAGL-PA_111_Sch_Y-PA_111_Sch_Y13_005_Nr_647" "https://memobase.ch/record/fad-001-DON3196",
"fad-001-DON3196"
), ),
TestParam( TestParam(
"test-2", "test-2",
2, 2,
"kafkaTest1.yml", "kafkaTest1.yml",
"Burgerbib-Krebser-208576",
"Burgerbib-Krebser-208576" "Burgerbib-Krebser-208576"
), ),
TestParam( TestParam(
"test-3", "test-3",
3, 3,
"kafkaTest1.yml", "kafkaTest1.yml",
"LS-film-162354",
"LS-film-162354" "LS-film-162354"
) )
) )
......
...@@ -21,5 +21,6 @@ data class TestParam( ...@@ -21,5 +21,6 @@ data class TestParam(
val name: String, val name: String,
val count: Int, val count: Int,
val settingsFileName: String, val settingsFileName: String,
val inputKey: String val inputKey: String,
val outputKey: String
) )
{"title":[{"de":[],"fr":[],"it":[],"un":["Antonia Bozzini con marito e figli davanti a un fondale appeso; dietro da sinistra: Angiolina, Pio, Clemente, Luigi, Giovanni e Anna; davanti con i genitori: Veronica e Caterina"]}],"type":{"name":{"de":["Fotografie"],"fr":["Photographie"],"it":["Fotografia"],"un":[]},"filter":"Foto"},"sourceID":"DON3196","abstract":[{"de":[],"fr":[],"it":[],"un":["Famiglia con fondale, tela, drappo, addobbo - Famiglia davanti ad un edificio, ad un muro"]}],"rightsHolder":[{"de":[],"fr":[],"it":[],"un":["Fondazione Archivio Roberto Donetta; Corzoneso"]}],"format":[{"displayLabel":{"de":[],"fr":[],"it":[],"un":[]},"name":{"de":["Fotoplatte"],"fr":["plaque photographique"],"it":["lastra fotografica"],"un":[]}}],"keywords":{"de":[],"fr":[],"it":[],"un":["Sedia , Cassa , Fondale scuro , Muro , Donna , Uomo , Bambini , Ragazzi ,"]},"personSubject":[{"name":{"de":[],"fr":[],"it":[],"un":["BOZZINI, Anna"]},"filter":"BOZZINI, Anna","facet":["0~B~","1~B~BOZZINI, Anna~"]}],"personCreator":[{"name":{"de":[],"fr":[],"it":[],"un":["DONETTA, Roberto"]},"relation":{"de":[],"fr":[],"it":[],"un":["Fotografo"]},"filter":"DONETTA, Roberto","facet":["0~D~","1~D~DONETTA, Roberto~"]}],"personsFacet":{"facet":["0~B~","1~B~BOZZINI, Anna~","0~D~","1~D~DONETTA, Roberto~"],"filter":["BOZZINI, Anna","DONETTA, Roberto"]},"placeFacet":{},"dateCreated":[{"date":"1900/1932","sort":"1900/1932","facet":["0~20.Jahrhundert~","1~20.Jahrhundert~1891-1900~","1~20.Jahrhundert~1901-1910~","1~20.Jahrhundert~1911-1920~","1~20.Jahrhundert~1921-1930~","1~20.Jahrhundert~1931-1940~"]}],"institution":[{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["fad"]}],"recordSet":{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["fad-001"]},"memoriavClaim":true,"locator":"https://media.memobase.k8s.unibas.ch/memo/fad-001-DON3196-1","accessDigital":[{"name":{"de":["Online"],"fr":["En ligne"],"it":["Online"],"un":[]},"filter":"public"}],"usageDigital":["http://rightsstatements.org/vocab/CNE/1.0/"],"usageDigitalGroup":[{"name":{"de":["Unklar (Copyright nicht bekannt)"],"fr":[""],"it":[""],"un":[]},"filter":"http://rightsstatements.org/vocab/CNE/1.0/"}],"digital":{},"callNumber":["DON3196"],"accessPhysical":[{"name":{"de":["vor Ort"],"fr":["Sur place"],"it":["Nel local"],"un":[]},"filter":"onsite"}],"usagePhysical":["http://rightsstatements.org/vocab/CNE/1.0/"],"physicalCharacteristics":[{"de":[],"fr":[],"it":[],"un":["Höhe: 18 cm","Weite: 13 cm","Bemerkung: Stato di conservazione : Buono"]}],"colourPhysical":[{"de":[],"fr":[],"it":[],"un":["Bianco e nero"]}],"access":[{"name":{"de":["Online"],"fr":["En ligne"],"it":["Online"],"un":[]},"filter":"public"},{"name":{"de":["vor Ort"],"fr":["Sur place"],"it":["Nel local"],"un":[]},"filter":"onsite"}],"published":false,"suggest":{"title":["Antonia Bozzini con marito e figli davanti a un fondale appeso; dietro da sinistra: Angiolina, Pio, Clemente, Luigi, Giovanni e Anna; davanti con i genitori: Veronica e Caterina"],"keywords":["Sedia , Cassa , Fondale scuro , Muro , Donna , Uomo , Bambini , Ragazzi ,"]},"id":"LAGL-PA_111_Sch_Y-PA_111_Sch_Y13_005_Nr_647"} {"title":[{"de":[],"fr":[],"it":[],"un":["Antonia Bozzini con marito e figli davanti a un fondale appeso; dietro da sinistra: Angiolina, Pio, Clemente, Luigi, Giovanni e Anna; davanti con i genitori: Veronica e Caterina"]}],"type":{"name":{"de":["Fotografie"],"fr":["Photographie"],"it":["Fotografia"],"un":[]},"filter":"Foto"},"sourceID":"DON3196","abstract":[{"de":[],"fr":[],"it":[],"un":["Famiglia con fondale, tela, drappo, addobbo - Famiglia davanti ad un edificio, ad un muro"]}],"rightsHolder":[{"de":[],"fr":[],"it":[],"un":["Fondazione Archivio Roberto Donetta; Corzoneso"]}],"format":[{"displayLabel":{"de":[],"fr":[],"it":[],"un":[]},"name":{"de":["Fotoplatte"],"fr":["plaque photographique"],"it":["lastra fotografica"],"un":[]}}],"keywords":{"de":[],"fr":[],"it":[],"un":["Sedia , Cassa , Fondale scuro , Muro , Donna , Uomo , Bambini , Ragazzi ,"]},"personSubject":[{"name":{"de":[],"fr":[],"it":[],"un":["BOZZINI, Anna"]},"filter":"BOZZINI, Anna","facet":["0~B~","1~B~BOZZINI, Anna~"]}],"personCreator":[{"name":{"de":[],"fr":[],"it":[],"un":["DONETTA, Roberto"]},"relation":{"de":[],"fr":[],"it":[],"un":["Fotografo"]},"filter":"DONETTA, Roberto","facet":["0~D~","1~D~DONETTA, Roberto~"]}],"personsFacet":{"facet":["0~B~","1~B~BOZZINI, Anna~","0~D~","1~D~DONETTA, Roberto~"],"filter":["BOZZINI, Anna","DONETTA, Roberto"]},"placeFacet":{},"dateCreated":[{"date":"1900/1932","sort":"1900/1932","facet":["0~20.Jahrhundert~","1~20.Jahrhundert~1891-1900~","1~20.Jahrhundert~1901-1910~","1~20.Jahrhundert~1911-1920~","1~20.Jahrhundert~1921-1930~","1~20.Jahrhundert~1931-1940~"]}],"institution":[{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["fad"]}],"recordSet":{"name":{"de":[],"fr":[],"it":[],"un":[]},"facet":["fad-001"]},"memoriavClaim":true,"locator":"https://media.memobase.k8s.unibas.ch/memo/fad-001-DON3196-1","accessDigital":[{"name":{"de":["Online"],"fr":["En ligne"],"it":["Online"],"un":[]},"filter":"public"}],"usageDigital":["http://rightsstatements.org/vocab/CNE/1.0/"],"usageDigitalGroup":[{"name":{"de":["Unklar (Copyright nicht bekannt)"],"fr":[""],"it":[""],"un":[]},"filter":"http://rightsstatements.org/vocab/CNE/1.0/"}],"digital":{},"callNumber":["DON3196"],"accessPhysical":[{"name":{"de":["vor Ort"],"fr":["Sur place"],"it":["Nel local"],"un":[]},"filter":"onsite"}],"usagePhysical":["http://rightsstatements.org/vocab/CNE/1.0/"],"physicalCharacteristics":[{"de":[],"fr":[],"it":[],"un":["Höhe: 18 cm","Weite: 13 cm","Bemerkung: Stato di conservazione : Buono"]}],"colourPhysical":[{"de":[],"fr":[],"it":[],"un":["Bianco e nero"]}],"access":[{"name":{"de":["Online"],"fr":["En ligne"],"it":["Online"],"un":[]},"filter":"public"},{"name":{"de":["vor Ort"],"fr":["Sur place"],"it":["Nel local"],"un":[]},"filter":"onsite"}],"published":false,"suggest":{"title":["Antonia Bozzini con marito e figli davanti a un fondale appeso; dietro da sinistra: Angiolina, Pio, Clemente, Luigi, Giovanni e Anna; davanti con i genitori: Veronica e Caterina"],"keywords":["Sedia , Cassa , Fondale scuro , Muro , Donna , Uomo , Bambini , Ragazzi ,"]},"id":"fad-001-DON3196"}
\ No newline at end of file \ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment