Commit 59dedb0d authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Update version

Fix reporting.
parent d116c6d3
Pipeline #23084 passed with stages
in 4 minutes and 5 seconds
...@@ -33,7 +33,7 @@ ext { ...@@ -33,7 +33,7 @@ ext {
dependencies { dependencies {
implementation 'ch.memobase:memobase-kafka-utils:0.2.3' implementation 'ch.memobase:memobase-kafka-utils:0.2.3'
implementation 'org.memobase:memobase-service-utilities:2.0.5' implementation 'org.memobase:memobase-service-utilities:2.0.5'
implementation 'ch.memobase:mapper-service-configuration:1.0.8' implementation 'ch.memobase:mapper-service-configuration:1.0.9'
// Logging Framework // Logging Framework
implementation "org.apache.logging.log4j:log4j-api:${log4jV}" implementation "org.apache.logging.log4j:log4j-api:${log4jV}"
implementation "org.apache.logging.log4j:log4j-core:${log4jV}" implementation "org.apache.logging.log4j:log4j-core:${log4jV}"
......
...@@ -180,7 +180,7 @@ class KafkaTopology( ...@@ -180,7 +180,7 @@ class KafkaTopology(
Report( Report(
key, key,
ReportStatus.fatal, ReportStatus.fatal,
"Error Messages: " + value.errorMessages.joinToString("\n"), "No Record Id Found: " + value.errorMessages.joinToString("\n"),
Service.step Service.step
).toJson() ).toJson()
} }
...@@ -204,7 +204,7 @@ class KafkaTopology( ...@@ -204,7 +204,7 @@ class KafkaTopology(
Report( Report(
key, key,
ReportStatus.fatal, ReportStatus.fatal,
"Error Messages: " + value.errorMessages.joinToString("\n"), "No Record Type Value: " + value.errorMessages.joinToString("\n"),
Service.step Service.step
).toJson() ).toJson()
} }
...@@ -232,7 +232,7 @@ class KafkaTopology( ...@@ -232,7 +232,7 @@ class KafkaTopology(
Report( Report(
result.first, result.first,
if (builder.isFatal) ReportStatus.fatal else ReportStatus.warning, if (builder.isFatal) ReportStatus.fatal else ReportStatus.warning,
"Error Messages: " + result.third.joinToString("\n"), "Write Record Error: " + result.third.joinToString("\n"),
Service.step Service.step
) )
) )
......
...@@ -18,6 +18,7 @@ package org.memobase ...@@ -18,6 +18,7 @@ package org.memobase
import ch.memobase.rdf.NS import ch.memobase.rdf.NS
import ch.memobase.reporting.Report import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import com.beust.klaxon.Klaxon import com.beust.klaxon.Klaxon
import org.apache.jena.rdf.model.ModelFactory import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.riot.Lang import org.apache.jena.riot.Lang
...@@ -43,6 +44,8 @@ import java.util.stream.Stream ...@@ -43,6 +44,8 @@ import java.util.stream.Stream
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
class IntegrationTests { class IntegrationTests {
private val klaxon = Klaxon()
private val resourcePath = "src/test/resources" private val resourcePath = "src/test/resources"
private fun readFile(fileName: String): String { private fun readFile(fileName: String): String {
return File("$resourcePath/$fileName").readText(Charset.defaultCharset()) return File("$resourcePath/$fileName").readText(Charset.defaultCharset())
...@@ -72,9 +75,9 @@ class IntegrationTests { ...@@ -72,9 +75,9 @@ class IntegrationTests {
val settings = service.settings val settings = service.settings
val headers = RecordHeaders() val headers = RecordHeaders()
headers.add(RecordHeader("sessionId", "test-session-id".toByteArray())) headers.add(RecordHeader("sessionId", "1".toByteArray()))
headers.add(RecordHeader("recordSetId", "test-record-set-id".toByteArray())) headers.add(RecordHeader("recordSetId", "ati-002".toByteArray()))
headers.add(RecordHeader("institutionId", "test-institution-id".toByteArray())) headers.add(RecordHeader("institutionId", "ati".toByteArray()))
headers.add(RecordHeader("isPublished", "false".toByteArray())) headers.add(RecordHeader("isPublished", "false".toByteArray()))
headers.add(RecordHeader("xmlRecordTag", "record".toByteArray())) headers.add(RecordHeader("xmlRecordTag", "record".toByteArray()))
headers.add(RecordHeader("xmlIdentifierFieldName", "identifierMain".toByteArray())) headers.add(RecordHeader("xmlIdentifierFieldName", "identifierMain".toByteArray()))
...@@ -93,7 +96,7 @@ class IntegrationTests { ...@@ -93,7 +96,7 @@ class IntegrationTests {
testDriver.pipeInput( testDriver.pipeInput(
factory.create( factory.create(
"import-process-config", "import-process-config",
"test-record-set-id#mapping", "ati-002#mapping",
readFile("kafkaTests/${params.count}/config/mapping.yml") readFile("kafkaTests/${params.count}/config/mapping.yml")
) )
) )
...@@ -104,6 +107,18 @@ class IntegrationTests { ...@@ -104,6 +107,18 @@ class IntegrationTests {
) )
) )
val reportRecord = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
val reportString = reportRecord.value()
val report = klaxon.parse<Report>(reportString)!!
if (report.status == ReportStatus.fatal) {
assertThat(report)
.isEqualTo(params.report)
return
}
val record = testDriver.readOutput( val record = testDriver.readOutput(
settings.outputTopic, settings.outputTopic,
...@@ -125,13 +140,6 @@ class IntegrationTests { ...@@ -125,13 +140,6 @@ class IntegrationTests {
val key = record.key() val key = record.key()
val sortedResult = sort(record.value().lines()).trim() val sortedResult = sort(record.value().lines()).trim()
val reportRecord = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
val reportString = reportRecord.value()
assertAll("", assertAll("",
{ assertThat(key).isEqualTo(params.expectedKey) }, { assertThat(key).isEqualTo(params.expectedKey) },
{ {
...@@ -147,17 +155,18 @@ class IntegrationTests { ...@@ -147,17 +155,18 @@ class IntegrationTests {
} }
private fun kafkaTests() = Stream.of( private fun kafkaTests() = Stream.of(
/* IntegrationTestParams( IntegrationTestParams(
1, 1,
"MEI_49884", "MEI_49884",
"https://memobase.ch/record/test-record-set-id-MEI_49884", "https://memobase.ch/record/test-record-set-id-MEI_49884",
Report( Report(
"https://memobase.ch/record/test-record-set-id-MEI_49884", "MEI_49884",
"SUCCESS", "FATAL",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-MEI_49884." "No Record Id Found: Found multiple values in the field 'identifierOriginal' for identifiers: 22861, 22861, 22861.",
Service.step
) )
), )/*,
IntegrationTestParams( IntegrationTestParams(
2, 2,
"Sig Han 1293", "Sig Han 1293",
...@@ -187,7 +196,7 @@ class IntegrationTests { ...@@ -187,7 +196,7 @@ class IntegrationTests {
"SUCCESS", "SUCCESS",
"Successfully mapped record with id https://memobase.ch/record/test-record-set-id-sigantur-example." "Successfully mapped record with id https://memobase.ch/record/test-record-set-id-sigantur-example."
) )
),*/ ),
IntegrationTestParams( IntegrationTestParams(
5, 5,
"ID_1", "ID_1",
...@@ -198,7 +207,7 @@ class IntegrationTests { ...@@ -198,7 +207,7 @@ class IntegrationTests {
"", "",
Service.step Service.step
) )
)/*, ),
IntegrationTestParams( IntegrationTestParams(
6, 6,
"ID_1", "ID_1",
......
record: record:
uri: original_id uri: identifierOriginal
type:
const: Foto
isSponsoredByMemoriav: true
title:
de: titel
descriptiveNote: # rico:descriptiveNote
de: beschreibung
scopeAndContent: # rico:scopeAndContent
de:
- entstehungsumstände
relation: related_document
identifiers: identifiers:
callNumber: original_id original: identifierOriginal
titles: # blank node rico:Title oldMemobase: identifierMain
main: # serial, broadcast type: objectType
de: titel title: title
titles:
- main: title
- series: SerieTitle
- series: SeriesTitle
- broadcast: BroadcastTitle
scopeAndContent: Context
sameAs: references
abstract: Abstract
descriptiveNote: Remarks
source: source
relation: relatedMaterial
isSponsoredByMemoriav: true
rights:
holder: rightsHolder
languages:
- content: language
- caption: captioningFormat
subject:
- prefLabel: Keywords
genre: genre:
- prefLabel: - prefLabel: genre
de: placeOfCapture:
- genre - name: RecordingLocation
editorialNote: relatedPlaces:
const: "Ursprungsfeld: Genre" - name: location
subject: # blank node skos:Concept with rico:hasSubject creationDate: createdDate
- prefLabel: issuedDate: issuedDate
de: temporal: PeriodOfTimeDate
- baufunktion creators:
editorialNote: - agent:
const: "Ursprungsfeld: Baufunktion" name: creatorPerson.name
placeOfCapture: # blank node rico:Place with relation rdau:P60556 "has place of capture" relationName: creatorPerson.role
name: - corporateBody:
de: aufnahmeort name: creatorCorporateBody.name
relationName: creatorCorporateBody.role
contributors:
- corporateBody:
name: contributorPerson.name
relationName: contributorPerson.role
- corporateBody:
name: contributorCorporateBody.name
relationName: contributorCorporateBody.role
producers:
- person:
name: producerPersonName
- corporateBody:
name: producerCorporateBodyName
relatedAgents:
- person:
name: FeaturedIndividual
- corporateBody:
name: FeaturedOrganisation
publishedBy: publishedBy:
- person:
name: publisherPerson.name
- corporateBody: - corporateBody:
name: name: publisherCorporateBody.name
de: publishedBy conditionsOfUse: rightsUsage
hasFindingAid: verwandte_dokumente # rdau:P60262 "has finding aid" (provisional field)
creationDate: erstellung # blank node rico:DateSet with rico:expressedDate
relatedPlaces: # blank node rico:Place with relation dcterms:spatial
- name:
de: quartier
- name:
de:
- strasse1
- hausnummer1
- name:
de:
- strasse2
- hausnummer2
- name:
de: bezeichnung_bauobjekt
physical: physical:
medium: # ebucore:hasMedium carrierType: medium
de: medium duration: duration
physicalCharacteristics: # rico:physicalCharacteristics physicalCharacteristics:
de: - prefix:
- format value: "Startposition: "
- orientation field: start
- zustand - prefix:
value: "Seitenverhältnis: "
field: videoFormatRatio
- prefix:
value: "TV Norm: "
field: videoFormatTVSystem
- prefix:
value: "Bemerkung: "
field: videoFormatRemarks
- prefix:
value: "Verfahren: "
field: imageFormatMode
- prefix:
value: "Bemerkung: "
field: imageFormatRemarks
- prefix:
value: "Abspielgeschwindigkeit: "
field: audioFormatPlaybackSpeed
- prefix:
value: "Bemerkung: "
field: audioFormatRemarks
- prefix:
value: "Weite: "
field: width
- prefix:
value: "Höhe: "
field: height
- prefix:
value: "Tonaufnahmeverfahren: "
field: audioTrackConfiguration
colour: colour:
- farbe # rdau:P60558 has colour content (provisional field) - videoFormatColor
descriptiveNote: #rico:descriptiveNote - videoFormatColorMode
de: - imageFormatColorMode
- bemerkungen - imageFormatColor
identifiers: # blank node rico:Identifier identifiers:
original: original_id callNumber: identifierCallNumber
\ No newline at end of file rights:
access: accessPhysical
usage:
name:
const: "Copyright Not Evaluated (CNE)"
sameAs:
const: "http://rightsstatements.org/vocab/CNE/1.0/"
digital:
locator: essenceLocator
rights:
access: accessDigital
usage:
name: rightsStatementText
sameAs: rightsStatementURL
\ No newline at end of file
{ {
"original_id": "MEI_49884" "title": [
"De Rosa con altri uomini",
"De Rosa con altri uomini",
"De Rosa con altri uomini"
],
"creatorPerson": [
{
"name": "Non identificato",
"role": "Author"
},
{
"name": "Non identificato",
"role": "Author"
},
{
"name": "Non identificato",
"role": "Author"
}
],
"Keywords": [
"Cronaca, avvenimenti, episodi storici / Guerra civile in Spagna",
"Cronaca, avvenimenti, episodi storici / Guerra civile in Spagna",
"Cronaca, avvenimenti, episodi storici / Guerra civile in Spagna"
],
"Context": [
"Fondo di provenienza: Associazione volontari alla Guerra di Spagna<br>",
"Fondo di provenienza: Associazione volontari alla Guerra di Spagna<br>",
"Fondo di provenienza: Associazione volontari alla Guerra di Spagna<br>"
],
"Abstract": [
"Veduta di quattro uomini in piedi in un bosco<br>Descrizione fisica dell'immagine: Originale<br>Iscrizioni presente sullaa fotografia: Sul recto del supporto sotto la fotografia, a penna: (\"De Rosa / prima della / morte sul / fronte di / Madrid agosto 36\")",
"Veduta di quattro uomini in piedi in un bosco<br>Descrizione fisica dell'immagine: Originale<br>Iscrizioni presente sullaa fotografia: Sul recto del supporto sotto la fotografia, a penna: (\"De Rosa / prima della / morte sul / fronte di / Madrid agosto 36\")",
"Veduta di quattro uomini in piedi in un bosco<br>Descrizione fisica dell'immagine: Originale<br>Iscrizioni presente sullaa fotografia: Sul recto del supporto sotto la fotografia, a penna: (\"De Rosa / prima della / morte sul / fronte di / Madrid agosto 36\")"
],
"Remarks": [
"Sul recto del supporto sotto la fotografia 07001, a penna: (\"De Rosa / prima della / morte sul / fronte di / Madrid\")",
"Sul recto del supporto sotto la fotografia 07001, a penna: (\"De Rosa / prima della / morte sul / fronte di / Madrid\")",
"Sul recto del supporto sotto la fotografia 07001, a penna: (\"De Rosa / prima della / morte sul / fronte di / Madrid\")"
],
"Claim": [
"Questo documento è stato salvaguardato grazie al sostegno di Memoriav.",
"Questo documento è stato salvaguardato grazie al sostegno di Memoriav.",
"Questo documento è stato salvaguardato grazie al sostegno di Memoriav."
],
"contributorPerson": [
{
"name": "Fondazione Pellegrini Canevascini",
"role": "Collettore"
},
{
"name": "Fondazione Pellegrini Canevascini",
"role": "Collettore"
},
{
"name": "Fondazione Pellegrini Canevascini",
"role": "Collettore"
}
],
"createdDate": [
"Agosto 1936",
"Agosto 1936",
"Agosto 1936"
],
"objectType": [
"Foto",
"Foto",
"Foto"
],
"medium": [
"Carta sensibile",
"Carta sensibile",
"Carta sensibile"
],
"imageFormatColorMode": [
"B/N",
"B/N",
"B/N"
],
"imageFormatMode": [
"Positivo",
"Positivo",
"Positivo"
],
"width": [
"145 mm",
"145 mm",
"145 mm"
],
"height": [
"105 mm",
"105 mm",
"105 mm"
],
"identifierOriginal": [
"22861",
"22861",
"22861"
],
"identifierCallNumber": [
"07000",
"07000",
"07000"
],
"identifierMain": [
"ASTi-22861",
"ASTi-22861",
"ASTi-22861"
],
"references": [
"http://www3.ti.ch/DECS/sw/struttura/decs/dcsu/ac/asti/cff/index.php?p=immagine&amp;id_immagine=22861",
"http://www3.ti.ch/DECS/sw/struttura/decs/dcsu/ac/asti/cff/index.php?p=immagine&amp;id_immagine=22861",
"http://www3.ti.ch/DECS/sw/struttura/decs/dcsu/ac/asti/cff/index.php?p=immagine&amp;id_immagine=22861"
],
"location": [
"Madrid - Spagna",
"Madrid - Spagna",
"Madrid - Spagna"
],
"rightsUsage": [
"Consultazione libera. Riproduzione subordinata all'autorizzazione della FPC",
"Consultazione libera. Riproduzione subordinata all'autorizzazione della FPC",
"Consultazione libera. Riproduzione subordinata all'autorizzazione della FPC"
],
"rightsHolder": [
"Fondazione Pellegrini Canevascini",
"Fondazione Pellegrini Canevascini",
"Fondazione Pellegrini Canevascini"
],
"accessPhysical": [
"onsite",
"onsite",
"onsite"
],
"accessDigital": [
"public",
"public",
"public"
],
"rightsStatementText": [
"Copyright Not Evaluated (CNE)",
"Copyright Not Evaluated (CNE)",
"Copyright Not Evaluated (CNE)"
],
"rightsStatementURL": [
"http://rightsstatements.org/vocab/CNE/1.0/",
"http://rightsstatements.org/vocab/CNE/1.0/",
"http://rightsstatements.org/vocab/CNE/1.0/"
]
} }
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment