Commit ad271354 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Upgraded lib dependency to 2.0.0

parent 8e6ce5bb
...@@ -34,7 +34,7 @@ ext { ...@@ -34,7 +34,7 @@ ext {
dependencies { dependencies {
compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.6.1' compile group: 'org.elasticsearch.client', name: 'elasticsearch-rest-high-level-client', version: '7.6.1'
implementation 'org.memobase:memobase-service-utilities:0.14.2' implementation 'org.memobase:memobase-service-utilities:2.0.0'
// Logging Framework // Logging Framework
implementation "org.apache.logging.log4j:log4j-api:${log4jV}" implementation "org.apache.logging.log4j:log4j-api:${log4jV}"
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.memobase package org.memobase
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.builders.* import org.memobase.builders.*
...@@ -26,7 +27,6 @@ import org.memobase.model.DocumentsSearchDoc ...@@ -26,7 +27,6 @@ import org.memobase.model.DocumentsSearchDoc
import org.memobase.model.EnrichedDigitalMetadata import org.memobase.model.EnrichedDigitalMetadata
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.model.Schema import org.memobase.model.Schema
import org.memobase.rdf.NS
class DocumentsSearchDocBuilder(private val mediaUrl: String) { class DocumentsSearchDocBuilder(private val mediaUrl: String) {
private val log = LogManager.getLogger("SearchDocTransform") private val log = LogManager.getLogger("SearchDocTransform")
......
...@@ -18,6 +18,7 @@ ...@@ -18,6 +18,7 @@
package org.memobase package org.memobase
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonArray import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import java.util.Properties import java.util.Properties
...@@ -31,7 +32,7 @@ import org.memobase.model.FacetContainer ...@@ -31,7 +32,7 @@ import org.memobase.model.FacetContainer
import org.memobase.model.InstitutionSearchDoc import org.memobase.model.InstitutionSearchDoc
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.model.Schema import org.memobase.model.Schema
import org.memobase.rdf.NS
class InstitutionSearchDocBuilder(path: String, appSettings: Properties) { class InstitutionSearchDocBuilder(path: String, appSettings: Properties) {
private val log = LogManager.getLogger("InstitutionSearchDocBuilder") private val log = LogManager.getLogger("InstitutionSearchDocBuilder")
......
...@@ -18,8 +18,12 @@ ...@@ -18,8 +18,12 @@
package org.memobase package org.memobase
import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.SettingsLoader
import com.fasterxml.jackson.databind.ObjectMapper import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.registerKotlinModule import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import java.io.StringWriter
import org.apache.kafka.streams.StreamsBuilder import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.Topology import org.apache.kafka.streams.Topology
import org.apache.kafka.streams.kstream.KStream import org.apache.kafka.streams.kstream.KStream
...@@ -28,12 +32,8 @@ import org.apache.logging.log4j.LogManager ...@@ -28,12 +32,8 @@ import org.apache.logging.log4j.LogManager
import org.memobase.helpers.Default import org.memobase.helpers.Default
import org.memobase.helpers.JSON import org.memobase.helpers.JSON
import org.memobase.helpers.KEYS import org.memobase.helpers.KEYS
import org.memobase.helpers.ReportStatus
import org.memobase.model.DocumentsSearchDoc import org.memobase.model.DocumentsSearchDoc
import org.memobase.model.Report
import org.memobase.model.Schema import org.memobase.model.Schema
import org.memobase.settings.SettingsLoader
import java.io.StringWriter
class KafkaTopology(private val settings: SettingsLoader) { class KafkaTopology(private val settings: SettingsLoader) {
private val log = LogManager.getLogger("SearchDocService") private val log = LogManager.getLogger("SearchDocService")
...@@ -43,7 +43,7 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -43,7 +43,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
private val searchDocTransform = DocumentsSearchDocBuilder(appSettings.getProperty(KEYS.SettingsProps.mediaUrl)) private val searchDocTransform = DocumentsSearchDocBuilder(appSettings.getProperty(KEYS.SettingsProps.mediaUrl))
private val institutionSearchDoc = private val institutionSearchDoc =
InstitutionSearchDocBuilder(appSettings.getProperty(KEYS.SettingsProps.institutionTypeLabelsPath), appSettings) InstitutionSearchDocBuilder(appSettings.getProperty(KEYS.SettingsProps.institutionTypeLabelsPath), appSettings)
private val jsonWriter = ObjectMapper().registerKotlinModule().writer() private val jsonWriter = ObjectMapper().registerKotlinModule().writer()
...@@ -51,68 +51,73 @@ class KafkaTopology(private val settings: SettingsLoader) { ...@@ -51,68 +51,73 @@ class KafkaTopology(private val settings: SettingsLoader) {
val builder = StreamsBuilder() val builder = StreamsBuilder()
val stream = builder.stream<String, String>(settings.inputTopic) val stream = builder.stream<String, String>(settings.inputTopic)
val branchedStream = stream val branchedStream = stream
.mapValues { value -> JSON.parse(value) } .mapValues { value -> JSON.parse(value) }
.filter { _, value -> value.isNotEmpty() } .filter { _, value -> value.isNotEmpty() }
.mapValues { value -> JSON.unpack(value) } .mapValues { value -> JSON.unpack(value) }
.branch( .branch(
Predicate { _, value -> value.containsKey(JSON.record) }, Predicate { _, value -> value.containsKey(JSON.record) },
Predicate { _, value -> value.containsKey(JSON.institution) }, Predicate { _, value -> value.containsKey(JSON.institution) },
Predicate { _, value -> value.containsKey(JSON.recordSet) }, Predicate { _, value -> value.containsKey(JSON.recordSet) },
Predicate { _, _ -> true } Predicate { _, _ -> true }
) )
val recordStream = branchedStream[0] val recordStream = branchedStream[0]
.mapValues { readOnlyKey, value -> .mapValues { readOnlyKey, value ->
try { try {
Pair( Pair(
searchDocTransform.transform(value), searchDocTransform.transform(value),
Report(readOnlyKey, "SUCCESS", "Transformed message into search doc.") Report(readOnlyKey, ReportStatus.success, "Transformed message into search doc.", Service.name)
) )
} catch (ex: InvalidInputException) { } catch (ex: InvalidInputException) {
Pair(DocumentsSearchDoc.DEFAULT, Report(readOnlyKey, "FAILURE", ex.localizedMessage)) Pair(DocumentsSearchDoc.DEFAULT, Report(readOnlyKey, ReportStatus.warning, ex.localizedMessage, Service.name))
}
} }
}
outputStreams(recordStream) outputStreams(recordStream)
val institutionStream = branchedStream[1] val institutionStream = branchedStream[1]
.mapValues { readOnlyKey, value -> .mapValues { readOnlyKey, value ->
try { try {
Pair( Pair(
institutionSearchDoc.transform(readOnlyKey, value), institutionSearchDoc.transform(readOnlyKey, value),
Report(readOnlyKey, "SUCCESS", "Transformed message into search doc.") Report(readOnlyKey, ReportStatus.success, "Transformed message into search doc.", Service.name)
) )
} catch (ex: InvalidInputException) { } catch (ex: InvalidInputException) {
Pair(Default.institutionSearchDoc, Report(readOnlyKey, "FAILURE", ex.localizedMessage)) Pair(Default.institutionSearchDoc, Report(
readOnlyKey,
ReportStatus.warning,
ex.localizedMessage,
Service.name))
}
} }
}
outputStreams(institutionStream) outputStreams(institutionStream)
branchedStream[3] branchedStream[3]
.mapValues { readOnlyKey, value -> .mapValues { readOnlyKey, value ->
Report( Report(
readOnlyKey, readOnlyKey,
ReportStatus.fatal, ReportStatus.fatal,
"No record, memobase institution or record set present in input data: $value." "No record, memobase institution or record set present in input data: $value.",
) Service.name
} )
.to(reportTopic) }
.to(reportTopic)
return builder.build() return builder.build()
} }
private fun outputStreams(stream: KStream<String, Pair<Schema, Report>>) { private fun outputStreams(stream: KStream<String, Pair<Schema, Report>>) {
stream stream
.mapValues { value -> value.second.toJson() } .mapValues { value -> value.second.toJson() }
.to(reportTopic) .to(reportTopic)
stream stream
.filterNot { _, value -> value.second.status == "FAILURE" } .filterNot { _, value -> value.second.status == "FAILURE" }
.mapValues { value -> value.first } .mapValues { value -> value.first }
.mapValues { value -> .mapValues { value ->
val out = StringWriter() val out = StringWriter()
jsonWriter.writeValue(out, value) jsonWriter.writeValue(out, value)
out.toString() out.toString()
} }
.to(settings.outputTopic) .to(settings.outputTopic)
} }
} }
...@@ -18,13 +18,16 @@ ...@@ -18,13 +18,16 @@
package org.memobase package org.memobase
import ch.memobase.settings.SettingsLoader
import kotlin.system.exitProcess import kotlin.system.exitProcess
import org.apache.kafka.streams.KafkaStreams import org.apache.kafka.streams.KafkaStreams
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS import org.memobase.helpers.KEYS
import org.memobase.settings.SettingsLoader
class Service(file: String = "app.yml") { class Service(file: String = "app.yml") {
companion object {
const val name = "search-doc-service"
}
private val log = LogManager.getLogger("SearchDocService") private val log = LogManager.getLogger("SearchDocService")
val settings = SettingsLoader( val settings = SettingsLoader(
......
...@@ -18,20 +18,20 @@ ...@@ -18,20 +18,20 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS
import org.memobase.helpers.Extract import org.memobase.helpers.Extract
import org.memobase.helpers.FacetBuildHelpers import org.memobase.helpers.FacetBuildHelpers
import org.memobase.helpers.KEYS
import org.memobase.model.AgentWithRelationContainer import org.memobase.model.AgentWithRelationContainer
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.rdf.NS
class AgentContainerBuilder( class AgentContainerBuilder(
private val identifiers: List<String>, private val identifiers: List<String>,
private val agentType: String, private val agentType: String,
private val creationRelationType: String?, private val creationRelationType: String?,
private val inputMap: Map<String, JsonObject> private val inputMap: Map<String, JsonObject>
) : IFieldBuilder { ) : IFieldBuilder {
private val log = LogManager.getLogger("AgentContainerBuilder") private val log = LogManager.getLogger("AgentContainerBuilder")
private val containers = mutableListOf<AgentWithRelationContainer>() private val containers = mutableListOf<AgentWithRelationContainer>()
...@@ -55,7 +55,7 @@ class AgentContainerBuilder( ...@@ -55,7 +55,7 @@ class AgentContainerBuilder(
override fun append(key: String, jsonObject: JsonObject): String { override fun append(key: String, jsonObject: JsonObject): String {
val names = Extract.languageContainer("agent-name", jsonObject[KEYS.name]) val names = Extract.languageContainer("agent-name", jsonObject[KEYS.name])
val name = if (names.isNotEmpty()) val name = if (names.isNotEmpty())
names.reduce { acc, languageContainer -> acc.merge(languageContainer) } names.reduce { acc, languageContainer -> acc.merge(languageContainer) }
else else
LanguageContainer(emptyList(), emptyList(), emptyList(), listOf("NoNameFound")) LanguageContainer(emptyList(), emptyList(), emptyList(), listOf("NoNameFound"))
log.warn("Creation relation without names in agent ${jsonObject["@id"]}") log.warn("Creation relation without names in agent ${jsonObject["@id"]}")
...@@ -82,12 +82,12 @@ class AgentContainerBuilder( ...@@ -82,12 +82,12 @@ class AgentContainerBuilder(
Pair(null, null) Pair(null, null)
} }
containers.add( containers.add(
AgentWithRelationContainer( AgentWithRelationContainer(
name, name,
relation, relation,
filter = filterAndFacet.first, filter = filterAndFacet.first,
facet = filterAndFacet.second facet = filterAndFacet.second
) )
) )
return "Created person container for person ${jsonObject[KEYS.entityId]}" return "Created person container for person ${jsonObject[KEYS.entityId]}"
} }
......
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonArray import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS
import org.memobase.helpers.DateFacetBuildHelpers import org.memobase.helpers.DateFacetBuildHelpers
import org.memobase.helpers.KEYS
import org.memobase.model.DateContainer import org.memobase.model.DateContainer
import org.memobase.rdf.NS
import java.lang.NumberFormatException
class DateContainerBuilder(private val containedIds: List<String>) : IFieldBuilder { class DateContainerBuilder(private val containedIds: List<String>) : IFieldBuilder {
private val log = LogManager.getLogger("DateContainerBuilder") private val log = LogManager.getLogger("DateContainerBuilder")
...@@ -62,13 +61,13 @@ class DateContainerBuilder(private val containedIds: List<String>) : IFieldBuild ...@@ -62,13 +61,13 @@ class DateContainerBuilder(private val containedIds: List<String>) : IFieldBuild
else -> emptyList() else -> emptyList()
} }
dateContainers.add( dateContainers.add(
DateContainer( DateContainer(
date = date, date = date,
sort = sort, sort = sort,
qualifier = qualifier, qualifier = qualifier,
certainty = certainty, certainty = certainty,
facet = facetList facet = facetList
) )
) )
return "Transformed date to date container." return "Transformed date to date container."
} }
......
...@@ -19,20 +19,18 @@ ...@@ -19,20 +19,18 @@
package org.memobase.builders package org.memobase.builders
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import com.fasterxml.jackson.module.kotlin.jacksonTypeRef
import org.memobase.helpers.Extract import org.memobase.helpers.Extract
import org.memobase.helpers.KEYS import org.memobase.helpers.KEYS
import org.memobase.model.EnrichedFacetContainer import org.memobase.model.EnrichedFacetContainer
import org.memobase.model.LanguageContainer import org.memobase.model.LanguageContainer
import org.memobase.rdf.NS
/** /**
* *
*/ */
class EnrichedFacetContainerBuilder( class EnrichedFacetContainerBuilder(
private val identifiers: List<String>, private val identifiers: List<String>,
private val targetType: String, private val targetType: String,
private val nameProperty: String private val nameProperty: String
) : IFieldBuilder { ) : IFieldBuilder {
private val containers = mutableListOf<EnrichedFacetContainer>() private val containers = mutableListOf<EnrichedFacetContainer>()
...@@ -46,10 +44,10 @@ class EnrichedFacetContainerBuilder( ...@@ -46,10 +44,10 @@ class EnrichedFacetContainerBuilder(
override fun append(key: String, jsonObject: JsonObject): String { override fun append(key: String, jsonObject: JsonObject): String {
val names = Extract.languageContainer(targetType, jsonObject[nameProperty]) val names = Extract.languageContainer(targetType, jsonObject[nameProperty])
.reduce { acc, languageContainer -> acc.merge(languageContainer) }.fillInEmpty() .reduce { acc, languageContainer -> acc.merge(languageContainer) }.fillInEmpty()
val ricoType = jsonObject[KEYS.ricoType] as String? val ricoType = jsonObject[KEYS.ricoType] as String?
containers.add( containers.add(
EnrichedFacetContainer(LanguageContainer.EMPTY, names, ricoType) EnrichedFacetContainer(LanguageContainer.EMPTY, names, ricoType)
) )
return "" return ""
} }
......
...@@ -18,20 +18,20 @@ ...@@ -18,20 +18,20 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.memobase.helpers.KEYS
import org.memobase.helpers.Extract import org.memobase.helpers.Extract
import org.memobase.helpers.KEYS
import org.memobase.model.FacetContainer import org.memobase.model.FacetContainer
import org.memobase.rdf.NS
/** /**
* @param identifiers: A list of identifiers to filter on. * @param identifiers: A list of identifiers to filter on.
*/ */
class FacettedContainerBuilder( class FacettedContainerBuilder(
private val identifiers: List<String>, private val identifiers: List<String>,
private val targetType: String, private val targetType: String,
private val nameProperty: String, private val nameProperty: String,
private val facetFunction: (jsonObject: JsonObject) -> Pair<String?, List<String>> private val facetFunction: (jsonObject: JsonObject) -> Pair<String?, List<String>>
) : IFieldBuilder { ) : IFieldBuilder {
private val containers = mutableListOf<FacetContainer>() private val containers = mutableListOf<FacetContainer>()
...@@ -44,12 +44,12 @@ class FacettedContainerBuilder( ...@@ -44,12 +44,12 @@ class FacettedContainerBuilder(
override fun append(key: String, jsonObject: JsonObject): String { override fun append(key: String, jsonObject: JsonObject): String {
val filterAndFacet = facetFunction(jsonObject) val filterAndFacet = facetFunction(jsonObject)
containers.add( containers.add(
FacetContainer( FacetContainer(
Extract.languageContainer("", jsonObject[nameProperty]) Extract.languageContainer("", jsonObject[nameProperty])
.reduce { acc, languageContainer -> acc.merge(languageContainer) }, .reduce { acc, languageContainer -> acc.merge(languageContainer) },
filter = filterAndFacet.first, filter = filterAndFacet.first,
facet = filterAndFacet.second facet = filterAndFacet.second
) )
) )
return "" return ""
} }
......
...@@ -15,7 +15,6 @@ ...@@ -15,7 +15,6 @@
* You should have received a copy of the GNU Affero General Public License * You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>. * along with this program. If not, see <https://www.gnu.org/licenses/>.
*/ */
package org.memobase.builders package org.memobase.builders
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
......
...@@ -18,11 +18,11 @@ ...@@ -18,11 +18,11 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS
import org.memobase.helpers.FacetBuildHelpers import org.memobase.helpers.FacetBuildHelpers
import org.memobase.rdf.NS import org.memobase.helpers.KEYS
class PersonFacetBuilder : IFieldBuilder { class PersonFacetBuilder : IFieldBuilder {
private val log = LogManager.getLogger("PersonFacetBuilder") private val log = LogManager.getLogger("PersonFacetBuilder")
......
...@@ -18,11 +18,11 @@ ...@@ -18,11 +18,11 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import org.memobase.helpers.KEYS
import org.memobase.helpers.FacetBuildHelpers import org.memobase.helpers.FacetBuildHelpers
import org.memobase.rdf.NS import org.memobase.helpers.KEYS
class PlaceFacetBuilder : IFieldBuilder { class PlaceFacetBuilder : IFieldBuilder {
private val log = LogManager.getLogger("PersonFacetBuilder") private val log = LogManager.getLogger("PersonFacetBuilder")
......
...@@ -18,11 +18,11 @@ ...@@ -18,11 +18,11 @@
package org.memobase.builders package org.memobase.builders
import ch.memobase.rdf.NS
import com.beust.klaxon.JsonObject import com.beust.klaxon.JsonObject
import org.memobase.helpers.KEYS
import org.memobase.helpers.Extract import org.memobase.helpers.Extract
import org.memobase.helpers.KEYS
import org.memobase.model.SuggestContainer import org.memobase.model.SuggestContainer
import org.memobase.rdf.NS
class SuggestContainerBuilder(private val hasSubjectIds: List<String>) : IFieldBuilder { class SuggestContainerBuilder(private val hasSubjectIds: List<String>) : IFieldBuilder {
...@@ -53,22 +53,22 @@ class SuggestContainerBuilder(private val hasSubjectIds: List<String>) : IFieldB ...@@ -53,22 +53,22 @@ class SuggestContainerBuilder(private val hasSubjectIds: List<String>) : IFieldB
return