Commit d4c6e433 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Update tests and logging

parent bd0df22c
...@@ -11,7 +11,7 @@ test: ...@@ -11,7 +11,7 @@ test:
tags: tags:
- mbr - mbr
script: script:
- gradle --no-daemon --no-scan --no-build-cache test --fail-fast --tests "org.memobase.Tests" - gradle --no-daemon --no-scan --no-build-cache test --fail-fast
.build-image: .build-image:
stage: publish stage: publish
......
...@@ -83,7 +83,7 @@ class SAXContentHandler(key: String, private val identifierFieldName: String, pr ...@@ -83,7 +83,7 @@ class SAXContentHandler(key: String, private val identifierFieldName: String, pr
currentElementContent += line currentElementContent += line
} }
else -> { else -> {
log.error(line) log.warn("Content without Tags: $line.")
} }
} }
} }
......
/* /*
* record-parser * xml-data-transform
* Copyright (C) 2019 Memobase * Copyright (C) 2019 Memobase
* *
* This program is free software: you can redistribute it and/or modify * This program is free software: you can redistribute it and/or modify
...@@ -17,28 +17,20 @@ ...@@ -17,28 +17,20 @@
*/ */
package org.memobase package org.memobase
import com.beust.klaxon.Klaxon
import java.io.File import java.io.File
import java.io.FileInputStream import java.io.FileInputStream
import java.nio.charset.Charset import java.nio.charset.Charset
import java.nio.file.Paths import java.nio.file.Paths
import java.util.stream.Stream
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.memobase.testing.EmbeddedSftpServer import org.memobase.testing.EmbeddedSftpServer
import java.io.InputStream import java.io.InputStream
import java.util.Properties import java.util.Properties
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
class Tests { class TestXsltTransform {
private val log = LogManager.getLogger("TestLogger") private val log = LogManager.getLogger("TestLogger")
private val resourcePath = "src/test/resources/data" private val resourcePath = "src/test/resources/data"
...@@ -47,96 +39,20 @@ class Tests { ...@@ -47,96 +39,20 @@ class Tests {
} }
private fun createInputStream(fileName: String): InputStream { private fun createInputStream(fileName: String): InputStream {
return File("src/test/resources/$fileName").inputStream() return File("$resourcePath/$fileName").inputStream()
}
private val sftpServer = EmbeddedSftpServer(22000, "user", "password")
init {
val files = listOf(1, 2).map { Pair("/memobase/test$it/data.xml", "$it/data.xml") }
for (pair in files) {
sftpServer.putFile(
Paths.get(pair.first, pair.second).toString(),
FileInputStream(Paths.get("src/test/resources/data", pair.second).toFile())
)
}
} }
@Test @Test
fun `test xslt tranform`() { fun `test xslt tranform`() {
val props = Properties() val props = Properties()
props.setProperty("xsltFilePath", "src/test/resources/test-transformer.xslt") props.setProperty("xsltFilePath", "${resourcePath}/test_transform/test-transformer.xslt")
props.setProperty("identifierFieldName", "identifierMain") props.setProperty("identifierFieldName", "identifierMain")
props.setProperty("recordTag", "record") props.setProperty("recordTag", "record")
val transformer = XMLTransformer(props) val transformer = XMLTransformer(props)
val result = transformer.applyXSLT("", createInputStream("test-transformer.xml")) val result = transformer.applyXSLT("", createInputStream("test_transform/test-transformer.xml"))
assertThat(result.key) assertThat(result.key)
.isEqualTo("ADG-102821") .isEqualTo("ADG-102821")
assertThat(result.value.output) assertThat(result.value.output.toString())
.isEqualTo("") .isEqualTo(readFile("test_transform/output.json"))
} }
/*
@ParameterizedTest
@MethodSource("testParams")
fun `test inputs`(params: TestParams) {
val service = Service("test${params.count}.yml")
val testDriver = TopologyTestDriver(service.topology, service.settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
)
testDriver.pipeInput(
factory.create(
service.settings.inputTopic, params.inputKey, readFile("${params.count}/input.json")
)
)
val record = testDriver.readOutput(
service.settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
assertThat(record)
.isNotNull
.hasFieldOrPropertyWithValue("key", params.outputKey)
.hasFieldOrPropertyWithValue("value", readFile("${params.count}/output.json"))
val reportedRecord = testDriver.readOutput(
"${service.settings.outputTopic}-reporting",
StringDeserializer(),
StringDeserializer()
)
if (reportedRecord != null) {
val data = reportedRecord.value()
val report = Klaxon().parse<Report>(data)
assertThat(report)
.isNotNull
.isEqualTo(params.expectedOutputReport)
} else {
log.error("No report for record $record.")
}
val processReport = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
assertThat(Klaxon().parse<Report>(processReport.value()))
.isNotNull
.isEqualTo(params.expectedOutputProcessReport)
}
private fun testParams() = Stream.of(
TestParams(
1,
"test-flat-input",
"key",
"key",
Report("", "", ""),
Report("", "", "")
)
)*/
} }
<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:fn="http://www.w3.org/2005/xpath-functions"
version="2.0"
xmlns:ns2="http://purl.org/dc/elements/1.1/"
xmlns:foxml="info:fedora/fedora-system:def/foxml#"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:audit="info:fedora/fedora-system:def/audit#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xmlns:fedora="info:fedora/fedora-system:def/relations-external#"
xmlns:fedora-model="info:fedora/fedora-system:def/model#"
xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xslt="http://www.w3.org/1999/XSL/Transform">
<xsl:output
indent="yes"
method="xml"
/>
<!--<xsl:template match="/digitalObject/datastream/datastreamVersion/xmlContent/ebuCoreMain/coreMetadata">
<xsl:copy-of select="."/>
</xsl:template>-->
<xsl:template match="coreMetadata">
<xsl:element name="record">
<!--<xsl:apply-templates select="test"/>
<xsl:apply-templates select="title, ns2:title, alternativeTitle, subject, description, format/essenceLocator, format/medium, type/*, references"/>-->
<xsl:apply-templates />
</xsl:element>
</xsl:template>
<!-- content of child node is copied to parent node -->
<xsl:template match="title | format/essenceLocator | format/duration | format/start | language | references | coverage/spatial/location">
<xsl:element name="{local-name()}">
<xsl:value-of select="child::*"/>
</xsl:element>
</xsl:template>
<!-- content of typeLabel is transformed to an element name and content of child node is copied to it-->
<xsl:template match="description[@typeLabel] | alternativeTitle[@typeLabel] | subject[@typeLabel]">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$typeLabel}">
<xsl:value-of select="child::*"/>
</xsl:element>
</xsl:template>
<!-- content of typeLabel is copied as content of the node -->
<xsl:template match="format/medium | format/dataFormat/captioningFormat | type/*">
<xsl:element name="{local-name()}">
<xsl:value-of select="@typeLabel | @language"/>
</xsl:element>
</xsl:template>
<xsl:template match="identifier">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$typeLabel}">
<xsl:value-of select="child::ns2:identifier"/>
</xsl:element>
</xsl:template>
<xsl:template match="format/videoFormat | format/imageFormat | format/audioFormat">
<xsl:variable name="format" select="local-name()"/>
<xsl:for-each select="technicalAttributeString">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$format}{$typeLabel}">
<xsl:value-of select="."/>
</xsl:element>
</xsl:for-each>
</xsl:template>
<!-- ToDo: role auslesen und dabei besitzende Insitution ausschliessen, producer auf Grund Rolle in eigenes Feld -->
<xsl:template match="contributor | creator | publisher">
<xsl:variable name="type" select="local-name()"/>
<xsl:for-each select=".">
<xsl:choose>
<xsl:when test="child::organisationDetails">
<xsl:element name="{$type}CorporateBody">
<xsl:element name="name">
<xsl:value-of select="descendant::organisationName"/>
</xsl:element>
<xsl:if test="child::role">
<xsl:element name="role">
<xsl:value-of select="child::role[@typeLabel]"/> <!-- funktioniert noch nicht -->
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>
<xsl:when test="child::contactDetails">
<xsl:element name="{$type}Person">
<xsl:element name="name">
<xsl:value-of select="descendant::name"/>
</xsl:element>
<xsl:if test="child::role">
<xsl:element name="role">
<xsl:value-of select="child::role[@typeLabel]"/> <!-- funktioniert noch nicht -->
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>
</xsl:choose>
</xsl:for-each>
</xsl:template>
<xsl:template match="rights[@typeLabel='Access']/ns2:rights">
<xsl:if test="matches(.,'onsite')">
<xsl:element name="accessPhsyical">
<xsl:text>onsite</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'noonsite')">
<xsl:element name="accessDigital">
<xsl:text>noonsite</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'public')">
<xsl:element name="accessDigital">
<xsl:text>public</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'private')">
<xsl:element name="accessDigital">
<xsl:text>private</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'faro')">
<xsl:element name="accessDigital">
<xsl:text>faro</xsl:text>
</xsl:element>
</xsl:if>
</xsl:template>
<xsl:template match="rights[@typeLabel='Holder']/rightsHolder/contactDetails/name">
<xsl:element name="rightsHolder">
<xsl:value-of select="."/>
</xsl:element>
</xsl:template>
<xsl:template match="date">
<xsl:for-each select=".">
<xsl:choose>
<xsl:when test="child::created">
<xsl:element name="dateCreated">
<xsl:call-template name="dates"/>
</xsl:element>
</xsl:when>
<xsl:when test="child::issued">
<xsl:element name="dateIssued">
<xsl:call-template name="dates"/>
</xsl:element>
</xsl:when>
</xsl:choose>
</xsl:for-each>
</xsl:template>
<!-- ToDo: ausbauen für alle attributes von date -->
<xsl:template name="dates">
<xsl:for-each select="child::*">
<xsl:value-of select="@startDate"/>
</xsl:for-each>
</xsl:template>
<!-- field to delete -->
<xsl:template match="isMemberOf | rights[@typeLabel='Usage']"/>
</xsl:stylesheet>
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ xml-data-transform
~ Copyright (C) 2020 Memoriav
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<root>
<creator>
<name>Sebastian</name>
<role>Autor</role>
</creator>
<date>1928</date>
</root>
{
"path": "/memobase/test1/data.xml",
"format": "XML"
}
\ No newline at end of file
{"title":"Grabung Castaneda (Siedlung)","SerieTitle":"Grabung Walo Burkart und Karl Keller-Tarnuzzer, Castaneda (Gräberfeld und Siedlung)","creatorPerson":{"name":"[unbekannt]"},"Keywords":"Ausgrabung, Archäologie","Abstract":"Feld 1, östlicher Teil von Westen","Claim":"Dieses Dokument wurde Dank der Unterstützung von Memoriav erhalten.","RecordingLocation":"Castaneda","contributorCorporateBody":{"name":"ADG"},"genre":"Grabungsfotografie / Arbeitsfotografie / Sachfotografie","objectType":"photograph","medium":"[keine Information vorhanden]","imageFormatColorMode":"sw","imageFormatRemarks":"Album-Nr.: A2","identifierOriginal":"102821","identifierCallNumber":"RM_1_33_6","identifierMain":"ADG-102821","location":"Schweiz, Graubünden, Castaneda","rightsHolder":"Archäologischer Dienst Graubünden","accessPhsyical":"onsite","accessDigital":"public"}
\ No newline at end of file
{
"title": "Grabung Castaneda (Siedlung)",
"SerieTitle": "Grabung Walo Burkart und Karl Keller-Tarnuzzer, Castaneda (Gräberfeld und Siedlung)",
"creatorPerson": {
"name": "[unbekannt]"
},
"Keywords": "Ausgrabung, Archäologie",
"Abstract": "Feld 1, östlicher Teil von Westen",
"Claim": "Dieses Dokument wurde Dank der Unterstützung von Memoriav erhalten.",
"RecordingLocation": "Castaneda",
"contributorCorporateBody": {
"name": "ADG"
},
"genre": "Grabungsfotografie / Arbeitsfotografie / Sachfotografie",
"objectType": "photograph",
"medium": "[keine Information vorhanden]",
"imageFormatColorMode": "sw",
"imageFormatRemarks": "Album-Nr.: A2",
"Original": "102821",
"CallNumber": "RM_1_33_6",
"Main": "ADG-102821",
"location": "Schweiz, Graubünden, Castaneda",
"rightsHolder": "Archäologischer Dienst Graubünden",
"accessPhsyical": "onsite",
"accessDigital": "public"
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment