Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

Commit 76845c21 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Combine update requests into one request for the entire document.

parent 8bbb8927
Pipeline #26631 passed with stages
in 4 minutes and 7 seconds
...@@ -18,6 +18,16 @@ ...@@ -18,6 +18,16 @@
package ch.memobase package ch.memobase
import ch.memobase.helpers.Constants.SettingsProps
import ch.memobase.helpers.ElasticSearchWrapper
import ch.memobase.helpers.JsonUtility
import ch.memobase.helpers.TranslationMappers
import ch.memobase.helpers.UpdateQueryBuilder
import ch.memobase.model.DocumentsSearchDoc
import ch.memobase.model.FacetContainer
import ch.memobase.model.InstitutionSearchDoc
import ch.memobase.model.RecordSetSearchDoc
import ch.memobase.model.Schema
import ch.memobase.reporting.Report import ch.memobase.reporting.Report
import ch.memobase.reporting.ReportStatus import ch.memobase.reporting.ReportStatus
import ch.memobase.settings.SettingsLoader import ch.memobase.settings.SettingsLoader
...@@ -30,17 +40,6 @@ import org.apache.kafka.streams.Topology ...@@ -30,17 +40,6 @@ import org.apache.kafka.streams.Topology
import org.apache.kafka.streams.kstream.KStream import org.apache.kafka.streams.kstream.KStream
import org.apache.kafka.streams.kstream.Predicate import org.apache.kafka.streams.kstream.Predicate
import org.apache.logging.log4j.LogManager import org.apache.logging.log4j.LogManager
import ch.memobase.helpers.Constants.SettingsProps
import ch.memobase.helpers.ElasticSearchWrapper
import ch.memobase.helpers.JsonUtility
import ch.memobase.helpers.TranslationMappers
import ch.memobase.helpers.UpdateQueryBuilder
import ch.memobase.model.DocumentsSearchDoc
import ch.memobase.model.FacetContainer
import ch.memobase.model.InstitutionSearchDoc
import ch.memobase.model.RecordSetSearchDoc
import ch.memobase.model.Schema
import ch.memobase.model.UpdateQuery
class KafkaTopology( class KafkaTopology(
private val settings: SettingsLoader, private val settings: SettingsLoader,
...@@ -56,9 +55,6 @@ class KafkaTopology( ...@@ -56,9 +55,6 @@ class KafkaTopology(
private val documentSearchDocBuilder = DocumentsSearchDocBuilder(translationMappers, elasticSearchWrapper, mediaUrl) private val documentSearchDocBuilder = DocumentsSearchDocBuilder(translationMappers, elasticSearchWrapper, mediaUrl)
private val institutionSearchDoc = InstitutionSearchDocBuilder(translationMappers, elasticSearchWrapper) private val institutionSearchDoc = InstitutionSearchDocBuilder(translationMappers, elasticSearchWrapper)
private val updateQueryBuilder = UpdateQueryBuilder()
private val recordSetSearchDocBuilder = private val recordSetSearchDocBuilder =
RecordSetSearchDocBuilder(elasticSearchWrapper) RecordSetSearchDocBuilder(elasticSearchWrapper)
...@@ -182,91 +178,57 @@ class KafkaTopology( ...@@ -182,91 +178,57 @@ class KafkaTopology(
} }
.to(settings.outputTopic) .to(settings.outputTopic)
// generate update messages for institution & record set names! // generate update messages for institution & record set names! Only need to check this for record sets.
schema schema
.map { key, value -> .filter { _, value -> value is RecordSetSearchDoc }
when (value) { .mapValues { value -> value as RecordSetSearchDoc }
is RecordSetSearchDoc -> { .mapValues { value ->
// Do not update if the name is already the same. val updateQueryBuilder = UpdateQueryBuilder(value.id)
val currentName = elasticSearchWrapper.getRecordSetName(value.id) val currentName = elasticSearchWrapper.getRecordSetName(value.id)
if (value.name == currentName) { if (value.name != currentName) {
log.info("No update for record set name for ${value.name} as they are already current.") log.info("Replace $currentName with ${value.name} for record set ${value.id}.")
KeyValue(key, null) updateQueryBuilder.updateRecordSetName(value.name)
} else { } else {
log.info("Replace $currentName with ${value.name} for record set ${value.id}.") log.info("No update for record set name for ${value.name} as they are already current.")
KeyValue(
"$key#update",
updateQueryBuilder.updateRecordSetName(value.id, value.name)
)
}
}
else -> KeyValue(key, null)
} }
Pair(updateQueryBuilder, value)
}
.mapValues { value ->
updateInstitutionContainer(value.second, value.first)
value
} }
.filter { _, value -> value != null } .map { key, value -> KeyValue("$key#update", value) }
.mapValues { value -> JsonUtility.queryToJson(value!!) } .mapValues { value -> JsonUtility.queryToJson(value.first.build()) }
.to(updateTopic) .to(updateTopic)
recordSetUpdate(schema, "masterInstitution")
recordSetUpdate(schema, "originalInstitution")
recordSetUpdate(schema, "accessInstitution")
recordSetUpdate(schema, "institution")
} }
private fun recordSetUpdate(stream: KStream<String, Schema>, targetField: String) { private fun updateInstitutionContainer(
stream.map { key, value -> recordSet: RecordSetSearchDoc,
when (value) { updateQueryBuilder: UpdateQueryBuilder
is RecordSetSearchDoc -> { ) {
val update = buildUpdateQuery(recordSet.id, recordSet.institution, "institution", updateQueryBuilder)
updateInstitutionContainer(value, targetField) buildUpdateQuery(recordSet.id, recordSet.masterInstitution, "masterInstitution", updateQueryBuilder)
if (update == null) { buildUpdateQuery(recordSet.id, recordSet.originalInstitution, "originalInstitution", updateQueryBuilder)
KeyValue(key, null) buildUpdateQuery(recordSet.id, recordSet.accessInstitution, "accessInstitution", updateQueryBuilder)
} else {
KeyValue(
"$key#update",
update
)
}
}
else -> KeyValue(key, null)
}
}
.filter { _, value -> value != null }
.mapValues { value -> JsonUtility.queryToJson(value!!) }
.to(updateTopic)
}
private fun updateInstitutionContainer(recordSet: RecordSetSearchDoc, targetField: String): UpdateQuery? {
return when (targetField) {
"institution" ->
buildUpdateQuery(recordSet.id, recordSet.institution, targetField)
"masterInstitution" ->
buildUpdateQuery(recordSet.id, recordSet.masterInstitution, targetField)
"originalInstitution" ->
buildUpdateQuery(recordSet.id, recordSet.originalInstitution, targetField)
"accessInstitution" ->
buildUpdateQuery(recordSet.id, recordSet.accessInstitution, targetField)
else -> throw Exception("Unknown institution type. Set the wrong constant somewhere...")
}
} }
private fun buildUpdateQuery( private fun buildUpdateQuery(
recordSetId: String, recordSetId: String,
newInstitutions: List<FacetContainer>, newInstitutions: List<FacetContainer>,
targetField: String targetField: String,
): UpdateQuery? { updateQueryBuilder: UpdateQueryBuilder
) {
val currentInstitutions = val currentInstitutions =
elasticSearchWrapper.getExtraInstitutionsFromRecordSet(recordSetId, targetField) elasticSearchWrapper.getExtraInstitutionsFromRecordSet(recordSetId, targetField)
return if (newInstitutions.containsAll(currentInstitutions) && return if (newInstitutions.containsAll(currentInstitutions) &&
currentInstitutions.containsAll(newInstitutions) currentInstitutions.containsAll(newInstitutions)
) { ) {
log.info("No update for $targetField for $recordSetId as they are already current.") log.info("No update for $targetField for $recordSetId as they are already current.")
null
} else { } else {
log.info("Updating field $targetField for $recordSetId with $newInstitutions.") log.info("Updating field $targetField for $recordSetId with $newInstitutions.")
updateQueryBuilder.updateInstitutionContainers( updateQueryBuilder.updateInstitutionContainers(
recordSetId,
targetField, targetField,
newInstitutions newInstitutions
) )
......
...@@ -21,30 +21,31 @@ import ch.memobase.model.FacetContainer ...@@ -21,30 +21,31 @@ import ch.memobase.model.FacetContainer
import ch.memobase.model.LanguageContainer import ch.memobase.model.LanguageContainer
import ch.memobase.model.UpdateQuery import ch.memobase.model.UpdateQuery
class UpdateQueryBuilder { class UpdateQueryBuilder(private val value: String) {
private val recordSetFacetTerm = "recordSet.facet" private val recordSetFacetTerm = "recordSet.facet"
private val sourceLines = mutableListOf<String>()
private val params = mutableMapOf<String, Any>()
fun updateRecordSetName(recordSet: String, names: LanguageContainer): UpdateQuery { fun updateRecordSetName(names: LanguageContainer) {
return UpdateQuery( sourceLines.add("ctx._source['recordSet']['name']['de'] = params.de")
recordSetFacetTerm, sourceLines.add("ctx._source['recordSet']['name']['fr'] = params.fr")
recordSet, sourceLines.add("ctx._source['recordSet']['name']['it'] = params.it")
"ctx._source['recordSet']['name']['de'] = params.de; ctx._source['recordSet']['name']['fr'] = params.fr; ctx._source['recordSet']['name']['it'] = params.it", params["de"] = names.de[0]
mapOf( params["fr"] = names.fr[0]
Pair("de", names.de[0]), params["it"] = names.it[0]
Pair("fr", names.fr[0]), }
Pair("it", names.it[0])
) fun updateInstitutionContainers(targetField: String, institutions: List<FacetContainer>) {
) sourceLines.add("ctx._source['${targetField}'] = params.${targetField}")
params[targetField] = institutions
} }
fun updateInstitutionContainers(recordSet: String, targetField: String, institutions: List<FacetContainer>): UpdateQuery { fun build(): UpdateQuery {
return UpdateQuery( return UpdateQuery(
recordSetFacetTerm, recordSetFacetTerm,
recordSet, value,
"ctx._source['${targetField}'] = params.containers", sourceLines.joinToString("; "),
mapOf( params
Pair("containers", institutions)
)
) )
} }
} }
\ No newline at end of file
...@@ -211,17 +211,18 @@ class TestRecordSetSearchDoc { ...@@ -211,17 +211,18 @@ class TestRecordSetSearchDoc {
val value = keyValue.value.replace(TestUtilities.dateRegex, "2020") val value = keyValue.value.replace(TestUtilities.dateRegex, "2020")
val resultValue = readFile("testAddingInstitutionUpdateOutput.json").replace(TestUtilities.dateRegex, "2020") val resultValue = readFile("testAddingInstitutionUpdateOutput.json").replace(TestUtilities.dateRegex, "2020")
val updates = updateTopicOutput.readKeyValuesToList() val updates = updateTopicOutput.readKeyValuesToList()[0]
assertAll("", assertAll("",
{ {
assertThat(value) assertThat(value)
.isEqualTo(resultValue) .isEqualTo(resultValue)
}, },
{ assertThat(updates).hasSize(5) }, {
assertThat(updates.value).isEqualTo(readFile("updateOutput.json"))
},
{
assertThat(updates.key).isEqualTo("testComplete#update")
},
{ assertThat(key).isEqualTo("testComplete") }, { assertThat(key).isEqualTo("testComplete") },
{ assertThat(reportKey).isEqualTo("testComplete") }, { assertThat(reportKey).isEqualTo("testComplete") },
{ {
......
{"params" : {"de": "New Name (DE)", "fr": "New Name (FR)", "it": "New Name (IT)", "institution": [{"facet" : [], "filter" : "holdInstitution1", "name" : {"de" : ["Hold Institution 1"], "fr" : ["Hold Institution 1"], "it" : ["Hold Institution 1"], "un" : ["Hold Institution 1"]}}, {"facet" : [], "filter" : "holdInstitution2", "name" : {"de" : ["Hold Institution 2"], "fr" : ["Hold Institution 2"], "it" : ["Hold Institution 2"], "un" : ["Hold Institution 2"]}}], "masterInstitution": [{"facet" : [], "filter" : "completeInstitution", "name" : {"de" : ["Complete Institution"], "fr" : ["Complete Institution"], "it" : ["Complete Institution"], "un" : ["Complete Institution"]}}], "originalInstitution": [{"facet" : [], "filter" : "completeInstitution", "name" : {"de" : ["Complete Institution"], "fr" : ["Complete Institution"], "it" : ["Complete Institution"], "un" : ["Complete Institution"]}}], "accessInstitution": [{"facet" : [], "filter" : "completeInstitution", "name" : {"de" : ["Complete Institution"], "fr" : ["Complete Institution"], "it" : ["Complete Institution"], "un" : ["Complete Institution"]}}]}, "source" : "ctx._source['recordSet']['name']['de'] = params.de; ctx._source['recordSet']['name']['fr'] = params.fr; ctx._source['recordSet']['name']['it'] = params.it; ctx._source['institution'] = params.institution; ctx._source['masterInstitution'] = params.masterInstitution; ctx._source['originalInstitution'] = params.originalInstitution; ctx._source['accessInstitution'] = params.accessInstitution", "term" : "recordSet.facet", "value" : "testComplete"}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment