Skip to content

GitLab

X
XML Data Transform
Commit d4c6e433 authored by Jonas Waeber's avatar Jonas Waeber
Browse files
Options

Update tests and logging

parent bd0df22c
Hide whitespace changes
Inline Side-by-side
Showing with 61 additions and 226 deletions
.gitlab-ci.yml
View file @ d4c6e433
......@@ -11,7 +11,7 @@ test:
tags:
- mbr
script:
- gradle --no-daemon --no-scan --no-build-cache test --fail-fast --tests "org.memobase.Tests"
- gradle --no-daemon --no-scan --no-build-cache test --fail-fast
.build-image:
stage: publish
......
src/main/kotlin/SAXContentHandler.kt
View file @ d4c6e433
......@@ -83,7 +83,7 @@ class SAXContentHandler(key: String, private val identifierFieldName: String, pr
currentElementContent += line
}
else -> {
log.error(line)
log.warn("Content without Tags: $line.")
}
}
}
......
src/test/kotlin/Tests.kt → src/test/kotlin/TestXsltTransform.kt
View file @ d4c6e433
/*
* record-parser
* xml-data-transform
* Copyright (C) 2019 Memobase
*
* This program is free software: you can redistribute it and/or modify
......@@ -17,28 +17,20 @@
*/
package org.memobase
import com.beust.klaxon.Klaxon
import java.io.File
import java.io.FileInputStream
import java.nio.charset.Charset
import java.nio.file.Paths
import java.util.stream.Stream
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.memobase.testing.EmbeddedSftpServer
import java.io.InputStream
import java.util.Properties
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class Tests {
class TestXsltTransform {
private val log = LogManager.getLogger("TestLogger")
private val resourcePath = "src/test/resources/data"
......@@ -47,96 +39,20 @@ class Tests {
}
private fun createInputStream(fileName: String): InputStream {
return File("src/test/resources/$fileName").inputStream()
}
private val sftpServer = EmbeddedSftpServer(22000, "user", "password")
init {
val files = listOf(1, 2).map { Pair("/memobase/test$it/data.xml", "$it/data.xml") }
for (pair in files) {
sftpServer.putFile(
Paths.get(pair.first, pair.second).toString(),
FileInputStream(Paths.get("src/test/resources/data", pair.second).toFile())
)
}
return File("$resourcePath/$fileName").inputStream()
}
@Test
fun `test xslt tranform`() {
val props = Properties()
props.setProperty("xsltFilePath", "src/test/resources/test-transformer.xslt")
props.setProperty("xsltFilePath", "${resourcePath}/test_transform/test-transformer.xslt")
props.setProperty("identifierFieldName", "identifierMain")
props.setProperty("recordTag", "record")
val transformer = XMLTransformer(props)
val result = transformer.applyXSLT("", createInputStream("test-transformer.xml"))
val result = transformer.applyXSLT("", createInputStream("test_transform/test-transformer.xml"))
assertThat(result.key)
.isEqualTo("ADG-102821")
assertThat(result.value.output)
.isEqualTo("")
assertThat(result.value.output.toString())
.isEqualTo(readFile("test_transform/output.json"))
}
/*
@ParameterizedTest
@MethodSource("testParams")
fun `test inputs`(params: TestParams) {
val service = Service("test${params.count}.yml")
val testDriver = TopologyTestDriver(service.topology, service.settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
)
testDriver.pipeInput(
factory.create(
service.settings.inputTopic, params.inputKey, readFile("${params.count}/input.json")
)
)
val record = testDriver.readOutput(
service.settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
assertThat(record)
.isNotNull
.hasFieldOrPropertyWithValue("key", params.outputKey)
.hasFieldOrPropertyWithValue("value", readFile("${params.count}/output.json"))
val reportedRecord = testDriver.readOutput(
"${service.settings.outputTopic}-reporting",
StringDeserializer(),
StringDeserializer()
)
if (reportedRecord != null) {
val data = reportedRecord.value()
val report = Klaxon().parse<Report>(data)
assertThat(report)
.isNotNull
.isEqualTo(params.expectedOutputReport)
} else {
log.error("No report for record $record.")
}
val processReport = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
assertThat(Klaxon().parse<Report>(processReport.value()))
.isNotNull
.isEqualTo(params.expectedOutputProcessReport)
}
private fun testParams() = Stream.of(
TestParams(
1,
"test-flat-input",
"key",
"key",
Report("", "", ""),
Report("", "", "")
)
)*/
}
src/test/resources/data/1/config/transform.xslt deleted 100644 → 0
View file @ bd0df22c
<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:fn="http://www.w3.org/2005/xpath-functions"
version="2.0"
xmlns:ns2="http://purl.org/dc/elements/1.1/"
xmlns:foxml="info:fedora/fedora-system:def/foxml#"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:audit="info:fedora/fedora-system:def/audit#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xmlns:fedora="info:fedora/fedora-system:def/relations-external#"
xmlns:fedora-model="info:fedora/fedora-system:def/model#"
xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xslt="http://www.w3.org/1999/XSL/Transform">
<xsl:output
indent="yes"
method="xml"
/>
<!--<xsl:template match="/digitalObject/datastream/datastreamVersion/xmlContent/ebuCoreMain/coreMetadata">
<xsl:copy-of select="."/>
</xsl:template>-->
<xsl:template match="coreMetadata">
<xsl:element name="record">
<!--<xsl:apply-templates select="test"/>
<xsl:apply-templates select="title, ns2:title, alternativeTitle, subject, description, format/essenceLocator, format/medium, type/*, references"/>-->
<xsl:apply-templates />
</xsl:element>
</xsl:template>
<!-- content of child node is copied to parent node -->
<xsl:template match="title | format/essenceLocator | format/duration | format/start | language | references | coverage/spatial/location">
<xsl:element name="{local-name()}">
<xsl:value-of select="child::*"/>
</xsl:element>
</xsl:template>
<!-- content of typeLabel is transformed to an element name and content of child node is copied to it-->
<xsl:template match="description[@typeLabel] | alternativeTitle[@typeLabel] | subject[@typeLabel]">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$typeLabel}">
<xsl:value-of select="child::*"/>
</xsl:element>
</xsl:template>
<!-- content of typeLabel is copied as content of the node -->
<xsl:template match="format/medium | format/dataFormat/captioningFormat | type/*">
<xsl:element name="{local-name()}">
<xsl:value-of select="@typeLabel | @language"/>
</xsl:element>
</xsl:template>
<xsl:template match="identifier">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$typeLabel}">
<xsl:value-of select="child::ns2:identifier"/>
</xsl:element>
</xsl:template>
<xsl:template match="format/videoFormat | format/imageFormat | format/audioFormat">
<xsl:variable name="format" select="local-name()"/>
<xsl:for-each select="technicalAttributeString">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$format}{$typeLabel}">
<xsl:value-of select="."/>
</xsl:element>
</xsl:for-each>
</xsl:template>
<!-- ToDo: role auslesen und dabei besitzende Insitution ausschliessen, producer auf Grund Rolle in eigenes Feld -->
<xsl:template match="contributor | creator | publisher">
<xsl:variable name="type" select="local-name()"/>
<xsl:for-each select=".">
<xsl:choose>
<xsl:when test="child::organisationDetails">
<xsl:element name="{$type}CorporateBody">
<xsl:element name="name">
<xsl:value-of select="descendant::organisationName"/>
</xsl:element>
<xsl:if test="child::role">
<xsl:element name="role">
<xsl:value-of select="child::role[@typeLabel]"/> <!-- funktioniert noch nicht -->
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>
<xsl:when test="child::contactDetails">
<xsl:element name="{$type}Person">
<xsl:element name="name">
<xsl:value-of select="descendant::name"/>
</xsl:element>
<xsl:if test="child::role">
<xsl:element name="role">
<xsl:value-of select="child::role[@typeLabel]"/> <!-- funktioniert noch nicht -->
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>
</xsl:choose>
</xsl:for-each>
</xsl:template>
<xsl:template match="rights[@typeLabel='Access']/ns2:rights">
<xsl:if test="matches(.,'onsite')">
<xsl:element name="accessPhsyical">
<xsl:text>onsite</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'noonsite')">
<xsl:element name="accessDigital">
<xsl:text>noonsite</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'public')">
<xsl:element name="accessDigital">
<xsl:text>public</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'private')">
<xsl:element name="accessDigital">
<xsl:text>private</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'faro')">
<xsl:element name="accessDigital">
<xsl:text>faro</xsl:text>
</xsl:element>
</xsl:if>
</xsl:template>
<xsl:template match="rights[@typeLabel='Holder']/rightsHolder/contactDetails/name">
<xsl:element name="rightsHolder">
<xsl:value-of select="."/>
</xsl:element>
</xsl:template>
<xsl:template match="date">
<xsl:for-each select=".">
<xsl:choose>
<xsl:when test="child::created">
<xsl:element name="dateCreated">
<xsl:call-template name="dates"/>
</xsl:element>
</xsl:when>
<xsl:when test="child::issued">
<xsl:element name="dateIssued">
<xsl:call-template name="dates"/>
</xsl:element>
</xsl:when>
</xsl:choose>
</xsl:for-each>
</xsl:template>
<!-- ToDo: ausbauen für alle attributes von date -->
<xsl:template name="dates">
<xsl:for-each select="child::*">
<xsl:value-of select="@startDate"/>
</xsl:for-each>
</xsl:template>
<!-- field to delete -->
<xsl:template match="isMemberOf | rights[@typeLabel='Usage']"/>
</xsl:stylesheet>
src/test/resources/data/1/data.xml deleted 100644 → 0
View file @ bd0df22c
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ xml-data-transform
~ Copyright (C) 2020 Memoriav
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<root>
<creator>
<name>Sebastian</name>
<role>Autor</role>
</creator>
<date>1928</date>
</root>
src/test/resources/data/1/input.json deleted 100644 → 0
View file @ bd0df22c
{
"path": "/memobase/test1/data.xml",
"format": "XML"
}
\ No newline at end of file
src/test/resources/data/test_transform/output.json 0 → 100644
View file @ d4c6e433
{"title":"Grabung Castaneda (Siedlung)","SerieTitle":"Grabung Walo Burkart und Karl Keller-Tarnuzzer, Castaneda (Gräberfeld und Siedlung)","creatorPerson":{"name":"[unbekannt]"},"Keywords":"Ausgrabung, Archäologie","Abstract":"Feld 1, östlicher Teil von Westen","Claim":"Dieses Dokument wurde Dank der Unterstützung von Memoriav erhalten.","RecordingLocation":"Castaneda","contributorCorporateBody":{"name":"ADG"},"genre":"Grabungsfotografie / Arbeitsfotografie / Sachfotografie","objectType":"photograph","medium":"[keine Information vorhanden]","imageFormatColorMode":"sw","imageFormatRemarks":"Album-Nr.: A2","identifierOriginal":"102821","identifierCallNumber":"RM_1_33_6","identifierMain":"ADG-102821","location":"Schweiz, Graubünden, Castaneda","rightsHolder":"Archäologischer Dienst Graubünden","accessPhsyical":"onsite","accessDigital":"public"}
\ No newline at end of file
src/test/resources/output.json deleted 100644 → 0
View file @ bd0df22c
{
"title": "Grabung Castaneda (Siedlung)",
"SerieTitle": "Grabung Walo Burkart und Karl Keller-Tarnuzzer, Castaneda (Gräberfeld und Siedlung)",
"creatorPerson": {
"name": "[unbekannt]"
},
"Keywords": "Ausgrabung, Archäologie",
"Abstract": "Feld 1, östlicher Teil von Westen",
"Claim": "Dieses Dokument wurde Dank der Unterstützung von Memoriav erhalten.",
"RecordingLocation": "Castaneda",
"contributorCorporateBody": {
"name": "ADG"
},
"genre": "Grabungsfotografie / Arbeitsfotografie / Sachfotografie",
"objectType": "photograph",
"medium": "[keine Information vorhanden]",
"imageFormatColorMode": "sw",
"imageFormatRemarks": "Album-Nr.: A2",
"Original": "102821",
"CallNumber": "RM_1_33_6",
"Main": "ADG-102821",
"location": "Schweiz, Graubünden, Castaneda",
"rightsHolder": "Archäologischer Dienst Graubünden",
"accessPhsyical": "onsite",
"accessDigital": "public"
}
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment