Skip to content

GitLab

Commit 03dfa568 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor tests to be more specific.

parent bf01c783
Pipeline #33534 failed with stages
in 1 minute and 48 seconds
Expand all Show whitespace changes
Inline Side-by-side
build.gradle
View file @ 03dfa568
......@@ -27,12 +27,12 @@ repositories {
ext {
kafkaV = '2.7.0'
log4jV = '2.11.2'
log4jV = '2.14.1'
}
dependencies {
implementation 'ch.memobase:memobase-kafka-utils:0.3.1'
implementation 'ch.memobase:mapper-service-configuration:1.1.0'
implementation 'ch.memobase:mapper-service-configuration:1.1.1'
implementation 'org.memobase:memobase-service-utilities:3.0.3'
implementation "org.apache.logging.log4j:log4j-core:${log4jV}"
......@@ -47,8 +47,8 @@ dependencies {
implementation "org.jetbrains.kotlin:kotlin-script-runtime:1.3.71"
implementation "org.jetbrains.kotlin:kotlin-reflect:1.3.71"
testImplementation("org.junit.jupiter:junit-jupiter:5.4.2")
testImplementation 'org.assertj:assertj-core:3.15.0'
testImplementation('org.junit.jupiter:junit-jupiter:5.8.0')
testImplementation 'org.assertj:assertj-core:3.20.2'
testImplementation group: 'org.apache.kafka', name: 'kafka-streams-test-utils', version: kafkaV
}
......
src/test/kotlin/IntegrationTests.kt deleted 100644 → 0
View file @ bf01c783
/*
* Mapper Service
* Copyright (C) 2020-2021 Memobase
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import ch.memobase.rdf.NS
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import com.beust.klaxon.Klaxon
import java.io.File
import java.io.FileOutputStream
import java.nio.charset.Charset
import java.util.stream.Stream
import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.riot.Lang
import org.apache.jena.riot.RDFDataMgr
import org.apache.jena.riot.RDFFormat
import org.apache.kafka.common.header.internals.RecordHeader
import org.apache.kafka.common.header.internals.RecordHeaders
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertAll
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.memobase.params.IntegrationTestParams
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class IntegrationTests {
private val klaxon = Klaxon()
private val resourcePath = "src/test/resources"
private fun readFile(fileName: String): String {
return File("$resourcePath/$fileName").readText(Charset.defaultCharset())
}
private val regex = Regex("(_:B[A-Za-z0-9]+)")
private val regexTime = Regex("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}")
private fun sort(source: List<String>): String {
return source.map {
var replacedString = it
for (matchResult in regex.findAll(it)) {
replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "_:B")
}
for (matchResult in regexTime.findAll(it)) {
replacedString = replacedString.replace(matchResult.groups[0]?.value.orEmpty(), "2020-10-10T09:10:22")
}
replacedString
}.sorted().reduce { acc, s -> acc + "\n" + s }.trim()
}
@ParameterizedTest
@MethodSource("kafkaTests")
fun `test kafka topology`(params: IntegrationTestParams) {
val service = Service("kafkaTest${params.count}.yml")
val output = readFile("kafkaTests/${params.count}/output.nt").lines()
val settings = service.settings
val headers = RecordHeaders()
headers.add(RecordHeader("sessionId", "1".toByteArray()))
headers.add(RecordHeader("recordSetId", "ati-002".toByteArray()))
headers.add(RecordHeader("institutionId", "ati".toByteArray()))
headers.add(RecordHeader("isPublished", "false".toByteArray()))
headers.add(RecordHeader("xmlRecordTag", "record".toByteArray()))
headers.add(RecordHeader("xmlIdentifierFieldName", "identifierMain".toByteArray()))
headers.add(RecordHeader("tableSheetIndex", "1".toByteArray()))
headers.add(RecordHeader("tableHeaderCount", "1".toByteArray()))
headers.add(RecordHeader("tableHeaderIndex", "1".toByteArray()))
headers.add(RecordHeader("tableIdentifierIndex", "1".toByteArray()))
val testDriver = TopologyTestDriver(KafkaTopology(settings).prepare().build(), settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
)
testDriver.pipeInput(
factory.create(
"mb-di-config-test",
"ati-002#mapping",
readFile("kafkaTests/${params.count}/config/mapping.yml")
)
)
testDriver.pipeInput(
factory.create(
settings.inputTopic, params.key, readFile("kafkaTests/${params.count}/input.json"), headers
)
)
val reportRecord = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
val reportString = reportRecord.value()
val report = klaxon.parse<Report>(reportString)!!
if (report.status == ReportStatus.fatal) {
assertThat(report)
.isEqualTo(params.report)
return
}
val record = testDriver.readOutput(
settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
val model = ModelFactory.createDefaultModel()
NS.prefixMapping.map {
model.setNsPrefix(it.key, it.value)
}
RDFDataMgr.read(model, record.value().byteInputStream(), Lang.NTRIPLES)
RDFDataMgr.write(
FileOutputStream("$resourcePath/kafkaTests/${params.count}/turtle-output.ttl"),
model,
RDFFormat.TURTLE_PRETTY
)
val key = record.key()
val sortedResult = sort(record.value().lines()).trim()
assertAll("",
{ assertThat(key).isEqualTo(params.expectedKey) },
{
assertThat(sortedResult)
.isEqualTo(sort(output).trim())
},
{
assertThat(Klaxon().parse<Report>(reportString))
.isEqualTo(params.report)
}
)
}
private fun kafkaTests() = Stream.of(
IntegrationTestParams(
1,
"MEI_49884",
"https://memobase.ch/record/test-record-set-id-MEI_49884",
Report(
"MEI_49884",
"FATAL",
"No Record Id Found: Found multiple values in the field 'identifierOriginal' for identifiers: 22861, 22861, 22861.",
"test"
)
)/*,
IntegrationTestParams(
2,
"Sig Han 1293",
"https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
Report(
"https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
"SUCCESS",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-Sig_Han_1293."
)
),
IntegrationTestParams(
3,
"Sig Han 1293",
"ttps://memobase.ch/record/TEST_RECORD_SET-Sig_Han_1293",
Report(
"https://memobase.ch/record/test-record-set-id-Sig_Han_1293",
"SUCCESS",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-Sig_Han_1293."
)
),
IntegrationTestParams(
4,
"InputKey",
"https://memobase.ch/record/test-record-set-id-sigantur-example",
Report(
"https://memobase.ch/record/test-record-set-id-sigantur-example",
"SUCCESS",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-sigantur-example."
)
),
IntegrationTestParams(
5,
"ID_1",
"https://memobase.ch/record/test-record-set-id-ID_1",
Report(
"https://memobase.ch/record/test-record-set-id-ID_1",
"SUCCESS",
"",
Service.step
)
),
IntegrationTestParams(
6,
"ID_1",
"https://memobase.ch/record/test-record-set-id-ID_1",
Report(
"https://memobase.ch/record/test-record-set-id-ID_1",
"SUCCESS",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
)
),
IntegrationTestParams(
7,
"ID_1",
"https://memobase.ch/record/test-record-set-id-ID_1",
Report(
"https://memobase.ch/record/test-record-set-id-ID_1",
"SUCCESS",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
)
),
IntegrationTestParams(
8,
"ID_1",
"https://memobase.ch/record/test-record-set-id-ID_1",
Report(
"https://memobase.ch/record/test-record-set-id-ID_1",
"SUCCESS",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
)
),
IntegrationTestParams(
9,
"ID_1",
"https://memobase.ch/record/test-record-set-id-ID_1",
Report(
"https://memobase.ch/record/test-record-set-id-ID_1",
"SUCCESS",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-ID_1."
)
)*/
)
}
src/test/kotlin/TestPipeline.kt
View file @ 03dfa568
......@@ -42,11 +42,10 @@ class TestPipeline {
}
private fun setupTest(
fileName: String,
count: Int,
inputKey: String
): Pair<TestOutputTopic<String, String>, TestOutputTopic<String, String>> {
val service = Service(fileName)
val service = Service("data/$count/app.yml")
val testDriver =
TopologyTestDriver(KafkaTopology(service.settings).prepare().build(), service.settings.kafkaStreamsSettings)
val inputValue = readFile("input.json", count)
......@@ -85,7 +84,7 @@ class TestPipeline {
@Test
fun `test 1 minimal configuration`() {
val topics = setupTest("app.yml", 1, "test-1")
val topics = setupTest(1, "test-1")
val record = topics.first.readRecord()
val recordValue = record.value
......@@ -106,6 +105,91 @@ class TestPipeline {
@Test
fun `test 2 minimal configuration with digital and physical object`() {
val topics = setupTest(2, "test-2")
val record = topics.first.readRecord()
val recordValue = record.value
val recordKey = record.key
assertAll(
{
assertThat(sort(recordValue))
.isEqualTo(sort(readFile("output.nt", 2)))
},
{
assertThat(recordKey)
.isEqualTo("https://memobase.ch/record/rst-001-1")
}
)
}
@Test
fun `test 3 test list input`() {
val topics = setupTest(3, "test-3")
assertThat(topics.second.isEmpty)
.withFailMessage("Reports topic is empty.")
.isFalse()
val report = topics.second.readRecord()
val reportKey = report.key
val reportValue = report.value
assertAll( "Assert reports.",
{
assertThat(reportKey)
.isEqualTo("https://memobase.ch/record/rst-001-22861")
},
{
assertThat(parseReport(reportValue))
.isEqualTo(Report("https://memobase.ch/record/rst-001-22861", "SUCCESS", "Generated RDF document.", "test"))
}
)
assertThat(topics.first.isEmpty)
.withFailMessage("Record Topic is empty")
.isFalse()
val record = topics.first.readRecord()
val recordValue = record.value
val recordKey = record.key
assertAll(
{
assertThat(sort(recordValue))
.isEqualTo(sort(readFile("output.nt", 3)))
},
{
assertThat(recordKey)
.isEqualTo("https://memobase.ch/record/rst-001-22861")
}
)
}
@Test
fun `test 4 if proxy field in mapping works`() {
val topics = setupTest(4, "test-4")
val report = topics.second.readRecord()
val reportKey = report.key
val reportValue = report.value
val record = topics.first.readRecord()
val recordValue = record.value
val recordKey = record.key
assertAll(
{
assertThat(sort(recordValue))
.isEqualTo(sort(readFile("output.nt", 4)))
},
{
assertThat(recordKey)
.isEqualTo("https://memobase.ch/record/rst-001-1")
}
)
}
}
\ No newline at end of file
src/test/kotlin/params/IntegrationTestParams.kt deleted 100644 → 0
View file @ bf01c783
/*
* mapper-service
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.params
import ch.memobase.reporting.Report
data class IntegrationTestParams(
val count: Int,
val key: String,
val expectedKey: String,
val report: Report
)
src/test/resources/app.yml → src/test/resources/data/1/app.yml
View file @ 03dfa568
......@@ -4,7 +4,7 @@ app:
kafka:
streams:
bootstrap.servers: localhost:12345
application.id: test-clinet-1234
application.id: test-client-1
topic:
in: test-topic-in
out: test-topic-out
......
src/test/resources/kafkaTest3.yml → src/test/resources/data/2/app.yml
View file @ 03dfa568
app:
configTopic: mb-di-config-test
configTopic: mb-di-config-topic
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
application.id: test-clinet-1234
application.id: test-client-2
topic:
in: test-topic-in
out: test-topic-out
process: test-topic-process
\ No newline at end of file
process: test-process
\ No newline at end of file
src/test/resources/data/2/input.json 0 → 100644
View file @ 03dfa568
{
"identifier": "1",
"type": "Film"
}
\ No newline at end of file
src/test/resources/data/2/mapping.yml 0 → 100644
View file @ 03dfa568
record:
uri: identifier
type: type
digital:
rights:
usage:
name:
const: In Copyright (InC)
sameAs:
const: https://rightsstatements.org/vocab/InC/1.0/
access:
const: public
\ No newline at end of file
src/test/resources/data/2/output.nt 0 → 100644
View file @ 03dfa568
<https://memobase.ch/digital/rst-001-1-1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/digital/rst-001-1-1> <https://memobase.ch/internal/proxyType> "proxydirect" .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#instantiates> <https://memobase.ch/record/rst-001-1> .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#type> "digitalObject" .
<https://memobase.ch/record/rst-001-1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/rst-001-1> <https://memobase.ch/internal/isPublished> "false"^^<http://www.w3.org/2001/XMLSchema#boolean> .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/digital/rst-001-1-1> .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#heldBy> <https://memobase.ch/institution/ins> .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#isPartOf> <https://memobase.ch/recordSet/rst-001> .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#type> "Film" .
_:B <http://schema.org/sameAs> "https://rightsstatements.org/vocab/InC/1.0/" .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Identifier> .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Rule> .
_:B <https://www.ica.org/standards/RiC/ontology#identifier> "rst-001-1" .
_:B <https://www.ica.org/standards/RiC/ontology#identifier> "rst-001-1-1" .
_:B <https://www.ica.org/standards/RiC/ontology#name> "In Copyright (InC)" .
_:B <https://www.ica.org/standards/RiC/ontology#name> "public" .
_:B <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/digital/rst-001-1-1> .
_:B <https://www.ica.org/standards/RiC/ontology#regulates> <https://memobase.ch/digital/rst-001-1-1> .
_:B <https://www.ica.org/standards/RiC/ontology#type> "access" .
_:B <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:B <https://www.ica.org/standards/RiC/ontology#type> "main" .
_:B <https://www.ica.org/standards/RiC/ontology#type> "usage" .
\ No newline at end of file
src/test/resources/kafkaTest4.yml → src/test/resources/data/3/app.yml
View file @ 03dfa568
app:
configTopic: mb-di-config-test
configTopic: mb-di-config-topic
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
application.id: test-clinet-1234
application.id: test-client-3
topic:
in: test-topic-in
out: test-topic-out
process: test-topic-process
\ No newline at end of file
process: test-process
\ No newline at end of file
src/test/resources/kafkaTests/1/input.json → src/test/resources/data/3/input.json
View file @ 03dfa568
......@@ -62,11 +62,7 @@
"Agosto 1936",
"Agosto 1936"
],
"objectType": [
"Foto",
"Foto",
"Foto"
],
"objectType": "Foto",
"medium": [
"Carta sensibile",
"Carta sensibile",
......@@ -92,16 +88,8 @@
"105 mm",
"105 mm"
],
"identifierOriginal": [
"22861",
"22861",
"22861"
],
"identifierCallNumber": [
"07000",
"07000",
"07000"
],
"identifierOriginal": "22861",
"identifierCallNumber": "0001",
"identifierMain": [
"ASTi-22861",
"ASTi-22861",
......
src/test/resources/data/3/output.nt 0 → 100644
View file @ 03dfa568
This diff is collapsed.
src/test/resources/kafkaTest2.yml → src/test/resources/data/4/app.yml
View file @ 03dfa568
app:
configTopic: mb-di-config-test
configTopic: mb-di-config-topic
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
application.id: test-clinet-1234
application.id: test-client-4
topic:
in: test-topic-in
out: test-topic-out
process: test-topic-process
\ No newline at end of file
process: test-process
\ No newline at end of file
src/test/resources/data/4/input.json 0 → 100644
View file @ 03dfa568
{
"identifier": "1",
"type": "Film"
}
\ No newline at end of file
src/test/resources/data/4/mapping.yml 0 → 100644
View file @ 03dfa568
record:
uri: identifier
type: type
digital:
proxy:
const: "proxy"
rights:
usage:
name:
const: In Copyright (InC)
sameAs:
const: https://rightsstatements.org/vocab/InC/1.0/
access:
const: public
\ No newline at end of file
src/test/resources/data/4/output.nt 0 → 100644
View file @ 03dfa568
<https://memobase.ch/digital/rst-001-1-1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Instantiation> .
<https://memobase.ch/digital/rst-001-1-1> <https://memobase.ch/internal/proxyType> "proxy" .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#instantiates> <https://memobase.ch/record/rst-001-1> .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#regulatedBy> _:B .
<https://memobase.ch/digital/rst-001-1-1> <https://www.ica.org/standards/RiC/ontology#type> "digitalObject" .
<https://memobase.ch/record/rst-001-1> <http://www.w3.org/1999/02/22-rdf-syntax-ns#type> <https://www.ica.org/standards/RiC/ontology#Record> .
<https://memobase.ch/record/rst-001-1> <https://memobase.ch/internal/isPublished> "false"^^<http://www.w3.org/2001/XMLSchema#boolean> .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#hasInstantiation> <https://memobase.ch/digital/rst-001-1-1> .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#heldBy> <https://memobase.ch/institution/ins> .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#identifiedBy> _:B .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#isPartOf> <https://memobase.ch/recordSet/rst-001> .
<https://memobase.ch/record/rst-001-1> <https://www.ica.org/standards/RiC/ontology#type> "Film" .