Commit 0b61b153 authored by Jonas Waeber's avatar Jonas Waeber

Update reporting.

parent 3bef121f
Pipeline #12318 passed with stages
in 5 minutes and 21 seconds
#Tue May 19 16:49:13 CEST 2020
distributionUrl=https\://services.gradle.org/distributions/gradle-6.3-all.zip
distributionBase=GRADLE_USER_HOME
distributionPath=GRADLE_USER_HOME
zipStorePath=wrapper/dists
zipStoreBase=GRADLE_USER_HOME
......@@ -67,18 +67,17 @@ class KafkaTopology(private val settings: SettingsLoader) {
// report full process as failure
errorFilter[0]
.mapValues { key, _ ->
Klaxon().toJsonString(
Report(
"xml-data-transform",
ReportStatus.failure,
ReportMessages.processFailure(key, "The input file is invalid.")
)
.mapValues { _ ->
ProcessReport(
"xml-data-transform",
ReportStatus.failure,
1,
0,
1
)
}
.to(settings.processReportTopic)
// TODO: Implement multi record xml documents splitter!
val transformedValue = errorFilter[1]
.mapValues { value -> sftpClient.open(File(value.path)) }
.map { key, value -> transformXml(key, value) }
......@@ -94,11 +93,30 @@ class KafkaTopology(private val settings: SettingsLoader) {
.to(settings.outputTopic)
stream
.mapValues { value -> value.report!!.toJson() }
.mapValues { value -> value.getReport().toJson() }
.to(reportingTopic)
stream
.mapValues { value -> value.report!!.toJson() }
.mapValues { value ->
val report = value.getReport()
if (report.status == ReportStatus.success) {
ProcessReport(
report.id,
ReportStatus.success,
1,
1,
0
)
} else {
ProcessReport(
report.id,
ReportStatus.failure,
1,
0,
1
)
}
}
.to(settings.processReportTopic)
}
......@@ -114,6 +132,4 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
}
/*
* xml-data-transform
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import com.beust.klaxon.Klaxon
data class ProcessReport(
val id: String,
val status: String,
val total: Int,
val successes: Int,
val failures: Int
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
}
/*
* sftp-reader
* xml-data-transform
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
......
......@@ -25,14 +25,26 @@ import org.xml.sax.ContentHandler
import org.xml.sax.Locator
import java.io.StringWriter
class SAXContentHandler(key: String, private val identifierFieldName: String, private val recordTag: String) : ContentHandler {
class SAXContentHandler(key: String, private val identifierFieldName: String, private val recordTag: String) :
ContentHandler {
private val log = LogManager.getLogger("SAXHandler")
val output = StringWriter()
var identifier: String = key
var report: Report? = null
private var report: Report? = null
private val jsonResult = JsonObject()
fun getReport(): Report {
return report.let {
it
?: Report(
identifier,
ReportStatus.failure,
"Unknown Failure: No report found."
)
}
}
private var currentElementTag: String = ""
private var currentInnerElementTag: String = ""
private var currentElementContent: String = ""
......
/*
* xml-data-transform
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase
import org.junit.jupiter.api.TestInstance
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestIntegration {
}
\ No newline at end of file
......@@ -24,5 +24,5 @@ data class TestParams(
val inputKey: String,
val outputKey: String,
val expectedOutputReport: Report,
val expectedOutputProcessReport: Report
val expectedOutputProcessReport: ProcessReport
)
......@@ -44,7 +44,7 @@ class TestXsltTransform {
}
@Test
fun `test old memobase xslt tranform`() {
fun `test old memobase xslt transform`() {
val props = Properties()
props.setProperty("xsltFilePath", "${resourcePath}/test_transform/test-transformer.xslt")
props.setProperty("identifierFieldName", "identifierMain")
......
This diff is collapsed.
<?xml version="1.0"?>
<!--
~ xml-data-transform
~ Copyright (C) 2020 Memoriav
~
~ This program is free software: you can redistribute it and/or modify
~ it under the terms of the GNU Affero General Public License as published by
~ the Free Software Foundation, either version 3 of the License, or
~ (at your option) any later version.
~
~ This program is distributed in the hope that it will be useful,
~ but WITHOUT ANY WARRANTY; without even the implied warranty of
~ MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
~ GNU Affero General Public License for more details.
~
~ You should have received a copy of the GNU Affero General Public License
~ along with this program. If not, see <https://www.gnu.org/licenses/>.
-->
<xsl:stylesheet version="1.0" xmlns:xsl="http://www.w3.org/1999/XSL/Transform">
</xsl:stylesheet>
\ No newline at end of file
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment