In order to mitigate against the brute force attacks against Gitlab accounts, we are moving to all edu-ID Logins. We would like to remind you to link your account with your edu-id. Login will be possible only by edu-ID after November 30, 2021. Here you can find the instructions for linking your account.

If you don't have a SWITCH edu-ID, you can create one with this guide here

kind regards

This Server has been upgraded to GitLab release 14.2.6

Commit d4c6e433 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Update tests and logging

parent bd0df22c
......@@ -11,7 +11,7 @@ test:
tags:
- mbr
script:
- gradle --no-daemon --no-scan --no-build-cache test --fail-fast --tests "org.memobase.Tests"
- gradle --no-daemon --no-scan --no-build-cache test --fail-fast
.build-image:
stage: publish
......
......@@ -83,7 +83,7 @@ class SAXContentHandler(key: String, private val identifierFieldName: String, pr
currentElementContent += line
}
else -> {
log.error(line)
log.warn("Content without Tags: $line.")
}
}
}
......
/*
* record-parser
* xml-data-transform
* Copyright (C) 2019 Memobase
*
* This program is free software: you can redistribute it and/or modify
......@@ -17,28 +17,20 @@
*/
package org.memobase
import com.beust.klaxon.Klaxon
import java.io.File
import java.io.FileInputStream
import java.nio.charset.Charset
import java.nio.file.Paths
import java.util.stream.Stream
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource
import org.memobase.testing.EmbeddedSftpServer
import java.io.InputStream
import java.util.Properties
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class Tests {
class TestXsltTransform {
private val log = LogManager.getLogger("TestLogger")
private val resourcePath = "src/test/resources/data"
......@@ -47,96 +39,20 @@ class Tests {
}
private fun createInputStream(fileName: String): InputStream {
return File("src/test/resources/$fileName").inputStream()
}
private val sftpServer = EmbeddedSftpServer(22000, "user", "password")
init {
val files = listOf(1, 2).map { Pair("/memobase/test$it/data.xml", "$it/data.xml") }
for (pair in files) {
sftpServer.putFile(
Paths.get(pair.first, pair.second).toString(),
FileInputStream(Paths.get("src/test/resources/data", pair.second).toFile())
)
}
return File("$resourcePath/$fileName").inputStream()
}
@Test
fun `test xslt tranform`() {
val props = Properties()
props.setProperty("xsltFilePath", "src/test/resources/test-transformer.xslt")
props.setProperty("xsltFilePath", "${resourcePath}/test_transform/test-transformer.xslt")
props.setProperty("identifierFieldName", "identifierMain")
props.setProperty("recordTag", "record")
val transformer = XMLTransformer(props)
val result = transformer.applyXSLT("", createInputStream("test-transformer.xml"))
val result = transformer.applyXSLT("", createInputStream("test_transform/test-transformer.xml"))
assertThat(result.key)
.isEqualTo("ADG-102821")
assertThat(result.value.output)
.isEqualTo("")
assertThat(result.value.output.toString())
.isEqualTo(readFile("test_transform/output.json"))
}
/*
@ParameterizedTest
@MethodSource("testParams")
fun `test inputs`(params: TestParams) {
val service = Service("test${params.count}.yml")
val testDriver = TopologyTestDriver(service.topology, service.settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
)
testDriver.pipeInput(
factory.create(
service.settings.inputTopic, params.inputKey, readFile("${params.count}/input.json")
)
)
val record = testDriver.readOutput(
service.settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
assertThat(record)
.isNotNull
.hasFieldOrPropertyWithValue("key", params.outputKey)
.hasFieldOrPropertyWithValue("value", readFile("${params.count}/output.json"))
val reportedRecord = testDriver.readOutput(
"${service.settings.outputTopic}-reporting",
StringDeserializer(),
StringDeserializer()
)
if (reportedRecord != null) {
val data = reportedRecord.value()
val report = Klaxon().parse<Report>(data)
assertThat(report)
.isNotNull
.isEqualTo(params.expectedOutputReport)
} else {
log.error("No report for record $record.")
}
val processReport = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
assertThat(Klaxon().parse<Report>(processReport.value()))
.isNotNull
.isEqualTo(params.expectedOutputProcessReport)
}
private fun testParams() = Stream.of(
TestParams(
1,
"test-flat-input",
"key",
"key",
Report("", "", ""),
Report("", "", "")
)
)*/
}
<?xml version="1.0" encoding="UTF-8"?>
<xsl:stylesheet xmlns:xsl="http://www.w3.org/1999/XSL/Transform"
xmlns:fn="http://www.w3.org/2005/xpath-functions"
version="2.0"
xmlns:ns2="http://purl.org/dc/elements/1.1/"
xmlns:foxml="info:fedora/fedora-system:def/foxml#"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
xmlns:audit="info:fedora/fedora-system:def/audit#"
xmlns:dc="http://purl.org/dc/elements/1.1/"
xmlns:oai_dc="http://www.openarchives.org/OAI/2.0/oai_dc/"
xmlns:rdf="http://www.w3.org/1999/02/22-rdf-syntax-ns#"
xmlns:rdfs="http://www.w3.org/2000/01/rdf-schema#"
xmlns:fedora="info:fedora/fedora-system:def/relations-external#"
xmlns:fedora-model="info:fedora/fedora-system:def/model#"
xmlns:xs="http://www.w3.org/2001/XMLSchema" xmlns:xslt="http://www.w3.org/1999/XSL/Transform">
<xsl:output
indent="yes"
method="xml"
/>
<!--<xsl:template match="/digitalObject/datastream/datastreamVersion/xmlContent/ebuCoreMain/coreMetadata">
<xsl:copy-of select="."/>
</xsl:template>-->
<xsl:template match="coreMetadata">
<xsl:element name="record">
<!--<xsl:apply-templates select="test"/>
<xsl:apply-templates select="title, ns2:title, alternativeTitle, subject, description, format/essenceLocator, format/medium, type/*, references"/>-->
<xsl:apply-templates />
</xsl:element>
</xsl:template>
<!-- content of child node is copied to parent node -->
<xsl:template match="title | format/essenceLocator | format/duration | format/start | language | references | coverage/spatial/location">
<xsl:element name="{local-name()}">
<xsl:value-of select="child::*"/>
</xsl:element>
</xsl:template>
<!-- content of typeLabel is transformed to an element name and content of child node is copied to it-->
<xsl:template match="description[@typeLabel] | alternativeTitle[@typeLabel] | subject[@typeLabel]">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$typeLabel}">
<xsl:value-of select="child::*"/>
</xsl:element>
</xsl:template>
<!-- content of typeLabel is copied as content of the node -->
<xsl:template match="format/medium | format/dataFormat/captioningFormat | type/*">
<xsl:element name="{local-name()}">
<xsl:value-of select="@typeLabel | @language"/>
</xsl:element>
</xsl:template>
<xsl:template match="identifier">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$typeLabel}">
<xsl:value-of select="child::ns2:identifier"/>
</xsl:element>
</xsl:template>
<xsl:template match="format/videoFormat | format/imageFormat | format/audioFormat">
<xsl:variable name="format" select="local-name()"/>
<xsl:for-each select="technicalAttributeString">
<xsl:variable name="typeLabel" select="@typeLabel"/>
<xsl:element name="{$format}{$typeLabel}">
<xsl:value-of select="."/>
</xsl:element>
</xsl:for-each>
</xsl:template>
<!-- ToDo: role auslesen und dabei besitzende Insitution ausschliessen, producer auf Grund Rolle in eigenes Feld -->
<xsl:template match="contributor | creator | publisher">
<xsl:variable name="type" select="local-name()"/>
<xsl:for-each select=".">
<xsl:choose>
<xsl:when test="child::organisationDetails">
<xsl:element name="{$type}CorporateBody">
<xsl:element name="name">
<xsl:value-of select="descendant::organisationName"/>
</xsl:element>
<xsl:if test="child::role">
<xsl:element name="role">
<xsl:value-of select="child::role[@typeLabel]"/> <!-- funktioniert noch nicht -->
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>
<xsl:when test="child::contactDetails">
<xsl:element name="{$type}Person">
<xsl:element name="name">
<xsl:value-of select="descendant::name"/>
</xsl:element>
<xsl:if test="child::role">
<xsl:element name="role">
<xsl:value-of select="child::role[@typeLabel]"/> <!-- funktioniert noch nicht -->
</xsl:element>
</xsl:if>
</xsl:element>
</xsl:when>
</xsl:choose>
</xsl:for-each>
</xsl:template>
<xsl:template match="rights[@typeLabel='Access']/ns2:rights">
<xsl:if test="matches(.,'onsite')">
<xsl:element name="accessPhsyical">
<xsl:text>onsite</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'noonsite')">
<xsl:element name="accessDigital">
<xsl:text>noonsite</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'public')">
<xsl:element name="accessDigital">
<xsl:text>public</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'private')">
<xsl:element name="accessDigital">
<xsl:text>private</xsl:text>
</xsl:element>
</xsl:if>
<xsl:if test="matches(.,'faro')">
<xsl:element name="accessDigital">
<xsl:text>faro</xsl:text>
</xsl:element>
</xsl:if>
</xsl:template>
<xsl:template match="rights[@typeLabel='Holder']/rightsHolder/contactDetails/name">
<xsl:element name="rightsHolder">
<xsl:value-of select="."/>
</xsl:element>
</xsl:template>
<xsl:template match="date">
<xsl:for-each select=".">
<xsl:choose>
<xsl:when test="child::created">
<xsl:element name="dateCreated">
<xsl:call-template name="dates"/>
</xsl:element>
</xsl:when>
<xsl:when test="child::issued">
<xsl:element name="dateIssued">
<xsl:call-template name="dates"/>
</xsl:element>
</xsl:when>
</xsl:choose>
</xsl:for-each>
</xsl:template>
<!-- ToDo: ausbauen für alle attributes von date -->
<xsl:template name="dates">
<xsl:for-each select="child::*">
<xsl:value-of select="@startDate"/>
</xsl:for-each>
</xsl:template>
<!-- field to delete -->
<xsl:template match="isMemberOf | rights[@typeLabel='Usage']"/>
</xsl:stylesheet>
<?xml version="1.0" encoding="UTF-8"?>
<!--
~ xml-data-transform
~ Copyright (C) 2020 Memoriav
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<root>
<creator>
<name>Sebastian</name>
<role>Autor</role>
</creator>
<date>1928</date>
</root>
{
"path": "/memobase/test1/data.xml",
"format": "XML"
}
\ No newline at end of file
{"title":"Grabung Castaneda (Siedlung)","SerieTitle":"Grabung Walo Burkart und Karl Keller-Tarnuzzer, Castaneda (Gräberfeld und Siedlung)","creatorPerson":{"name":"[unbekannt]"},"Keywords":"Ausgrabung, Archäologie","Abstract":"Feld 1, östlicher Teil von Westen","Claim":"Dieses Dokument wurde Dank der Unterstützung von Memoriav erhalten.","RecordingLocation":"Castaneda","contributorCorporateBody":{"name":"ADG"},"genre":"Grabungsfotografie / Arbeitsfotografie / Sachfotografie","objectType":"photograph","medium":"[keine Information vorhanden]","imageFormatColorMode":"sw","imageFormatRemarks":"Album-Nr.: A2","identifierOriginal":"102821","identifierCallNumber":"RM_1_33_6","identifierMain":"ADG-102821","location":"Schweiz, Graubünden, Castaneda","rightsHolder":"Archäologischer Dienst Graubünden","accessPhsyical":"onsite","accessDigital":"public"}
\ No newline at end of file
{
"title": "Grabung Castaneda (Siedlung)",
"SerieTitle": "Grabung Walo Burkart und Karl Keller-Tarnuzzer, Castaneda (Gräberfeld und Siedlung)",
"creatorPerson": {
"name": "[unbekannt]"
},
"Keywords": "Ausgrabung, Archäologie",
"Abstract": "Feld 1, östlicher Teil von Westen",
"Claim": "Dieses Dokument wurde Dank der Unterstützung von Memoriav erhalten.",
"RecordingLocation": "Castaneda",
"contributorCorporateBody": {
"name": "ADG"
},
"genre": "Grabungsfotografie / Arbeitsfotografie / Sachfotografie",
"objectType": "photograph",
"medium": "[keine Information vorhanden]",
"imageFormatColorMode": "sw",
"imageFormatRemarks": "Album-Nr.: A2",
"Original": "102821",
"CallNumber": "RM_1_33_6",
"Main": "ADG-102821",
"location": "Schweiz, Graubünden, Castaneda",
"rightsHolder": "Archäologischer Dienst Graubünden",
"accessPhsyical": "onsite",
"accessDigital": "public"
}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment