Commit ad271354 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Upgraded lib dependency to 2.0.0

parent 8e6ce5bb
...@@ -34,7 +34,7 @@ ext { ...@@ -34,7 +34,7 @@ ext {
dependencies { dependencies {
compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.6.1' compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.6.1'
implementation 'org.memobase:memobase-service-utilities:0.14.2' implementation 'org.memobase:memobase-service-utilities:2.0.0'
// Logging Framework // Logging Framework
implementation "org.apache.logging.log4j:log4j-api:${log4jV}" implementation "org.apache.logging.log4j:log4j-api:${log4jV}"
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.memobase package org.memobase
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.builders.* import org.memobase.builders.*
...@@ -26,7 +27,6 @@ import org.memobase.model.DocumentsSearchDoc ...@@ -26,7 +27,6 @@ import org.memobase.model.DocumentsSearchDoc
import org.memobase.model.EnrichedDigitalMetadata import org.memobase.model.EnrichedDigitalMetadata
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.model.Schema import org.memobase.model.Schema
import org.memobase.rdf.NS
class DocumentsSearchDocBuilder(private val mediaUrl: String) { class DocumentsSearchDocBuilder(private val mediaUrl: String) {
private val log = LogManager.getLogger("SearchDocTransform") private val log = LogManager.getLogger("SearchDocTransform")
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.memobase package org.memobase
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonArray import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import java.util.Properties import java.util.Properties
...@@ -31,7 +32,7 @@ import org.memobase.model.FacetContainer ...@@ -31,7 +32,7 @@ import org.memobase.model.FacetContainer
import org.memobase.model.InstitutionSearchDoc import org.memobase.model.InstitutionSearchDoc
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.model.Schema import org.memobase.model.Schema
import org.memobase.rdf.NS
class InstitutionSearchDocBuilder(path: String, appSettings: Properties) { class InstitutionSearchDocBuilder(path: String, appSettings: Properties) {
private val log = LogManager.getLogger("InstitutionSearchDocBuilder") private val log = LogManager.getLogger("InstitutionSearchDocBuilder")
......
...@@ -18,8 +18,12 @@ ...@@ -18,8 +18,12 @@
package org.memobase package org.memobase
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.SettingsLoader
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.registerKotlinModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import java.io.StringWriter
import org.apache.kafka.streams.StreamsBuilder import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.Topology import org.apache.kafka.streams.Topology
import org.apache.kafka.streams.kstream.KStream import org.apache.kafka.streams.kstream.KStream
...@@ -28,12 +32,8 @@ import org.apache.logging.log4j.LogManager ...@@ -28,12 +32,8 @@ import org.apache.logging.log4j.LogManager
import org.memobase.helpers.Default import org.memobase.helpers.Default
import org.memobase.helpers.JSON import org.memobase.helpers.JSON
import org.memobase.helpers.KEYS import org.memobase.helpers.KEYS
import org.memobase.helpers.ReportStatus
import org.memobase.model.DocumentsSearchDoc import org.memobase.model.DocumentsSearchDoc
import org.memobase.model.Report
import org.memobase.model.Schema import org.memobase.model.Schema
import org.memobase.settings.SettingsLoader
import java.io.StringWriter
class KafkaTopology(private val settings: SettingsLoader) { class KafkaTopology(private val settings: SettingsLoader) {
private val log = LogManager.getLogger("SearchDocService") private val log = LogManager.getLogger("SearchDocService")
...@@ -66,10 +66,10 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -66,10 +66,10 @@ class KafkaTopology(private val settings: SettingsLoader) {
try { try {
Pair( Pair(
searchDocTransform.transform(value), searchDocTransform.transform(value),
Report(readOnlyKey, "SUCCESS", "Transformed message into search doc.") Report(readOnlyKey, ReportStatus.success, "Transformed message into search doc.", Service.name)
) )
} catch (ex: InvalidInputException) { } catch (ex: InvalidInputException) {
Pair(DocumentsSearchDoc.DEFAULT, Report(readOnlyKey, "FAILURE", ex.localizedMessage)) Pair(DocumentsSearchDoc.DEFAULT, Report(readOnlyKey, ReportStatus.warning, ex.localizedMessage, Service.name))
} }
} }
...@@ -80,10 +80,14 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -80,10 +80,14 @@ class KafkaTopology(private val settings: SettingsLoader) {
try { try {
Pair( Pair(
institutionSearchDoc.transform(readOnlyKey, value), institutionSearchDoc.transform(readOnlyKey, value),
Report(readOnlyKey, "SUCCESS", "Transformed message into search doc.") Report(readOnlyKey, ReportStatus.success, "Transformed message into search doc.", Service.name)
) )
} catch (ex: InvalidInputException) { } catch (ex: InvalidInputException) {
Pair(Default.institutionSearchDoc, Report(readOnlyKey, "FAILURE", ex.localizedMessage)) Pair(Default.institutionSearchDoc, Report(
readOnlyKey,
ReportStatus.warning,
ex.localizedMessage,
Service.name))
} }
} }
outputStreams(institutionStream) outputStreams(institutionStream)
...@@ -93,7 +97,8 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -93,7 +97,8 @@ class KafkaTopology(private val settings: SettingsLoader) {
Report( Report(
readOnlyKey, readOnlyKey,
ReportStatus.fatal, ReportStatus.fatal,
"No record, memobase institution or record set present in input data: $value." "No record, memobase institution or record set present in input data: $value.",
Service.name
) )
} }
.to(reportTopic) .to(reportTopic)
......
...@@ -18,13 +18,16 @@ ...@@ -18,13 +18,16 @@
package org.memobase package org.memobase
import ch.memobase.settings.SettingsLoader
import kotlin.system.exitProcess import kotlin.system.exitProcess
import org.apache.kafka.streams.KafkaStreams import org.apache.kafka.streams.KafkaStreams
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS import org.memobase.helpers.KEYS
import org.memobase.settings.SettingsLoader
class Service(file: String = "app.yml") { class Service(file: String = "app.yml") {
companion object {
const val name = "search-doc-service"
}
private val log = LogManager.getLogger("SearchDocService") private val log = LogManager.getLogger("SearchDocService")
val settings = SettingsLoader( val settings = SettingsLoader(
......
...@@ -18,14 +18,14 @@ ...@@ -18,14 +18,14 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS
import org.memobase.helpers.Extract import org.memobase.helpers.Extract
import org.memobase.helpers.FacetBuildHelpers import org.memobase.helpers.FacetBuildHelpers
import org.memobase.helpers.KEYS
import org.memobase.model.AgentWithRelationContainer import org.memobase.model.AgentWithRelationContainer
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.rdf.NS
class AgentContainerBuilder( class AgentContainerBuilder(
private val identifiers: List<String>, private val identifiers: List<String>,
......
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonArray import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS
import org.memobase.helpers.DateFacetBuildHelpers import org.memobase.helpers.DateFacetBuildHelpers
import org.memobase.helpers.KEYS
import org.memobase.model.DateContainer import org.memobase.model.DateContainer
import org.memobase.rdf.NS
import java.lang.NumberFormatException
class DateContainerBuilder(private val containedIds: List<String>) : IFieldBuilder { class DateContainerBuilder(private val containedIds: List<String>) : IFieldBuilder {
private val log = LogManager.getLogger("DateContainerBuilder") private val log = LogManager.getLogger("DateContainerBuilder")
......
...@@ -19,12 +19,10 @@ ...@@ -19,12 +19,10 @@
package org.memobase.builders package org.memobase.builders
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import com.fasterxml.jackson.module.kotlin.jacksonTypeRef
import org.memobase.helpers.Extract import org.memobase.helpers.Extract
import org.memobase.helpers.KEYS import org.memobase.helpers.KEYS
import org.memobase.model.EnrichedFacetContainer import org.memobase.model.EnrichedFacetContainer
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.rdf.NS
/** /**
* *
......
...@@ -18,11 +18,11 @@ ...@@ -18,11 +18,11 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.memobase.helpers.KEYS
import org.memobase.helpers.Extract import org.memobase.helpers.Extract
import org.memobase.helpers.KEYS
import org.memobase.model.FacetContainer import org.memobase.model.FacetContainer
import org.memobase.rdf.NS
/** /**
* @param identifiers: A list of identifiers to filter on. * @param identifiers: A list of identifiers to filter on.
......
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
package org.memobase.builders package org.memobase.builders
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
......
...@@ -18,11 +18,11 @@ ...@@ -18,11 +18,11 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS
import org.memobase.helpers.FacetBuildHelpers import org.memobase.helpers.FacetBuildHelpers
import org.memobase.rdf.NS import org.memobase.helpers.KEYS
class PersonFacetBuilder : IFieldBuilder { class PersonFacetBuilder : IFieldBuilder {
private val log = LogManager.getLogger("PersonFacetBuilder") private val log = LogManager.getLogger("PersonFacetBuilder")
......
...@@ -18,11 +18,11 @@ ...@@ -18,11 +18,11 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS
import org.memobase.helpers.FacetBuildHelpers import org.memobase.helpers.FacetBuildHelpers
import org.memobase.rdf.NS import org.memobase.helpers.KEYS
class PlaceFacetBuilder : IFieldBuilder { class PlaceFacetBuilder : IFieldBuilder {
private val log = LogManager.getLogger("PersonFacetBuilder") private val log = LogManager.getLogger("PersonFacetBuilder")
......
...@@ -18,11 +18,11 @@ ...@@ -18,11 +18,11 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.memobase.helpers.KEYS
import org.memobase.helpers.Extract import org.memobase.helpers.Extract
import org.memobase.helpers.KEYS
import org.memobase.model.SuggestContainer import org.memobase.model.SuggestContainer
import org.memobase.rdf.NS
class SuggestContainerBuilder(private val hasSubjectIds: List<String>) : IFieldBuilder { class SuggestContainerBuilder(private val hasSubjectIds: List<String>) : IFieldBuilder {
......
package org.memobase.helpers package org.memobase.helpers
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonArray import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import com.beust.klaxon.Klaxon import com.beust.klaxon.Klaxon
import com.beust.klaxon.KlaxonException import com.beust.klaxon.KlaxonException
import org.apache.logging.log4j.LogManager
import org.memobase.rdf.NS
import java.io.StringReader import java.io.StringReader
import org.apache.logging.log4j.LogManager
object JSON { object JSON {
......
package org.memobase.helpers package org.memobase.helpers
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.model.FacetContainer import org.memobase.model.FacetContainer
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.rdf.NS
/** /**
Extraction helpers for institutions and record sets. Extraction helpers for institutions and record sets.
......
package org.memobase.helpers
object ReportStatus {
const val success = "SUCCESS"
const val ignored = "IGNORED"
const val warning = "WARNING"
const val fatal = "FATAL"
}
\ No newline at end of file
/*
* Media Linker
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package org.memobase.model
import com.beust.klaxon.Klaxon
import java.time.LocalDateTime
data class Report(
val id: String,
val status: String,
val message: String,
val step: String = "search-doc-service",
val timestamp: String = LocalDateTime.now().toString()
) {
fun toJson(): String {
return Klaxon().toJsonString(this)
}
override fun equals(other: Any?): Boolean {
if (this === other) return true
if (javaClass != other?.javaClass) return false
other as Report
if (id != other.id) return false
if (status != other.status) return false
if (message != other.message) return false
if (step != other.step) return false
return true
}
override fun hashCode(): Int {
var result = id.hashCode()
result = 31 * result + status.hashCode()
result = 31 * result + message.hashCode()
result = 31 * result + step.hashCode()
return result
}
}
...@@ -17,23 +17,19 @@ ...@@ -17,23 +17,19 @@
*/ */
package org.memobase package org.memobase
import com.beust.klaxon.Klaxon import java.io.File
import java.nio.charset.Charset
import java.util.stream.Stream
import org.apache.kafka.common.serialization.StringDeserializer import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.assertj.core.api.Assertions.assertThat import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertAll
import org.junit.jupiter.params.ParameterizedTest import org.junit.jupiter.params.ParameterizedTest
import org.junit.jupiter.params.provider.MethodSource import org.junit.jupiter.params.provider.MethodSource
import org.memobase.model.Report
import org.memobase.params.TestParam import org.memobase.params.TestParam
import java.io.File
import java.nio.charset.Charset
import java.util.stream.Stream
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
class IntegrationTest { class IntegrationTest {
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.memobase package org.memobase
import ch.memobase.rdf.NS
import com.beust.klaxon.json import com.beust.klaxon.json
import java.util.stream.Stream import java.util.stream.Stream
import org.assertj.core.api.Assertions import org.assertj.core.api.Assertions
...@@ -32,7 +33,6 @@ import org.memobase.helpers.DateFacetBuildHelpers ...@@ -32,7 +33,6 @@ import org.memobase.helpers.DateFacetBuildHelpers
import org.memobase.helpers.KEYS import org.memobase.helpers.KEYS
import org.memobase.params.PersonFacetBuilderParams import org.memobase.params.PersonFacetBuilderParams
import org.memobase.params.TestDate import org.memobase.params.TestDate
import org.memobase.rdf.NS
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestFacetBuilders { class TestFacetBuilders {
......
package org.memobase package org.memobase
import com.beust.klaxon.Klaxon import ch.memobase.reporting.Report
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.registerKotlinModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import java.io.File
import java.nio.charset.Charset
import org.apache.kafka.common.serialization.StringDeserializer import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver import org.apache.kafka.streams.TopologyTestDriver
...@@ -11,13 +13,6 @@ import org.assertj.core.api.Assertions.assertThat ...@@ -11,13 +13,6 @@ import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertAll import org.junit.jupiter.api.assertAll
import org.memobase.model.InstitutionSearchDoc
import org.memobase.model.Report
import java.io.File
import java.io.StringReader
import java.io.StringWriter
import java.nio.charset.Charset
import kotlin.reflect.typeOf
@TestInstance(TestInstance.Lifecycle.PER_CLASS) @TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestInstitutionSearchDoc { class TestInstitutionSearchDoc {
...@@ -76,7 +71,8 @@ class TestInstitutionSearchDoc { ...@@ -76,7 +71,8 @@ class TestInstitutionSearchDoc {
Report( Report(
"https://memobase.ch/institution/RadioX", "https://memobase.ch/institution/RadioX",
"SUCCESS", "SUCCESS",
"Transformed message into search doc." "Transformed message into search doc.",
Service.name
) )
) )
} }
......
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment