Commit 11497cb3 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Fix institution update failing.

Add test for institution change
parent fec48d57
Pipeline #26568 passed with stages
in 4 minutes and 34 seconds
......@@ -36,6 +36,7 @@ import org.memobase.helpers.JsonUtility
import org.memobase.helpers.TranslationMappers
import org.memobase.helpers.UpdateQueryBuilder
import org.memobase.model.DocumentsSearchDoc
import org.memobase.model.FacetContainer
import org.memobase.model.InstitutionSearchDoc
import org.memobase.model.RecordSetSearchDoc
import org.memobase.model.Schema
......@@ -187,11 +188,12 @@ class KafkaTopology(
when (value) {
is RecordSetSearchDoc -> {
// Do not update if the name is already the same.
if (value.name == elasticSearchWrapper.getRecordSetName(value.id)) {
val currentName = elasticSearchWrapper.getRecordSetName(value.id)
if (value.name == currentName) {
log.info("No update for record set name for ${value.name} as they are already current.")
KeyValue(key, null)
} else {
log.info("Updating names for record set ${value.id} in documents.")
log.info("Replace $currentName with ${value.name} for record set ${value.id}.")
KeyValue(
"$key#update",
updateQueryBuilder.updateRecordSetName(value.id, value.name)
......@@ -237,70 +239,37 @@ class KafkaTopology(
private fun updateInstitutionContainer(recordSet: RecordSetSearchDoc, targetField: String): UpdateQuery? {
return when (targetField) {
"institution" -> {
if (recordSet.institution.containsAll(
elasticSearchWrapper.getExtraInstitutionsFromRecordSet(recordSet.id, targetField)
)
) {
log.info("No update for $targetField for ${recordSet.id} as they are already current.")
null
} else {
log.info("Updating field $targetField for ${recordSet.id} with ${recordSet.institution}.")
updateQueryBuilder.updateInstitutionContainers(
recordSet.id,
targetField,
recordSet.institution
)
}
}
"masterInstitution" -> {
if (recordSet.masterInstitution.containsAll(
elasticSearchWrapper.getExtraInstitutionsFromRecordSet(recordSet.id, targetField)
)
) {
log.info("No update for $targetField for ${recordSet.id} as they are already current.")
null
} else {
log.info("Updating field $targetField for ${recordSet.id} with ${recordSet.masterInstitution}.")
updateQueryBuilder.updateInstitutionContainers(
recordSet.id,
targetField,
recordSet.masterInstitution
)
}
}
"originalInstitution" -> {
if (recordSet.originalInstitution.containsAll(
elasticSearchWrapper.getExtraInstitutionsFromRecordSet(recordSet.id, targetField)
)
) {
log.info("No update for $targetField for ${recordSet.id} as they are already current.")
null
} else {
log.info("Updating field $targetField for ${recordSet.id} with ${recordSet.originalInstitution}.")
updateQueryBuilder.updateInstitutionContainers(
recordSet.id,
targetField, recordSet.originalInstitution
)
}
}
"accessInstitution" -> {
if (recordSet.accessInstitution.containsAll(
elasticSearchWrapper.getExtraInstitutionsFromRecordSet(recordSet.id, targetField)
)
) {
log.info("No update for $targetField for ${recordSet.id} as they are already current.")
null
} else {
log.info("Updating field $targetField for ${recordSet.id} with ${recordSet.accessInstitution}.")
updateQueryBuilder.updateInstitutionContainers(
recordSet.id,
targetField,
recordSet.accessInstitution
)
}
}
"institution" ->
buildUpdateQuery(recordSet.id, recordSet.institution, targetField)
"masterInstitution" ->
buildUpdateQuery(recordSet.id, recordSet.masterInstitution, targetField)
"originalInstitution" ->
buildUpdateQuery(recordSet.id, recordSet.originalInstitution, targetField)
"accessInstitution" ->
buildUpdateQuery(recordSet.id, recordSet.accessInstitution, targetField)
else -> throw Exception("Unknown institution type. Set the wrong constant somewhere...")
}
}
private fun buildUpdateQuery(
recordSetId: String,
newInstitutions: List<FacetContainer>,
targetField: String
): UpdateQuery? {
val currentInstitutions =
elasticSearchWrapper.getExtraInstitutionsFromRecordSet(recordSetId, targetField)
return if (newInstitutions.containsAll(currentInstitutions) &&
currentInstitutions.containsAll(newInstitutions)
) {
log.info("No update for $targetField for $recordSetId as they are already current.")
null
} else {
log.info("Updating field $targetField for $recordSetId with $newInstitutions.")
updateQueryBuilder.updateInstitutionContainers(
recordSetId,
targetField,
newInstitutions
)
}
}
}
......@@ -26,9 +26,9 @@ class TestElasticSearchWrapper {
private val log = LogManager.getLogger(TestElasticSearchWrapper::class.java)
private val host = "localhost"
private val port = 8080
private val documentsIndex = "documents-v20"
private val institutionIndex = "institutions-v5"
private val recordSetIndex = "record-sets-v5"
private val documentsIndex = "documents-v21"
private val institutionIndex = "institutions-v6"
private val recordSetIndex = "record-sets-v6"
private val client: RestHighLevelClient = connect()
......@@ -129,7 +129,9 @@ class TestElasticSearchWrapper {
)
}
)
} @Test
}
@Test
@Disabled
fun `test get original institutions from aag-001`() {
val wrapper = ElasticSearchWrapper(
......@@ -162,4 +164,30 @@ class TestElasticSearchWrapper {
}
)
}
@Test
@Disabled
fun `test contains all`() {
val source = listOf(
FacetContainer(
LanguageContainer(
listOf("Schweizerisches Bundesarchiv"),
listOf("Archives fédérales suisses"),
listOf("Archivio federale svizzero"),
emptyList()
), "bar", listOf()
)
)
val wrapper = ElasticSearchWrapper(
client,
TestUtilities.translationMappers,
documentsIndex,
institutionIndex,
recordSetIndex
)
val institutions = wrapper.getExtraInstitutionsFromRecordSet("bar-001", "institution")
assertThat(source.containsAll(institutions)).isFalse()
}
}
\ No newline at end of file
......@@ -31,5 +31,4 @@ class TestFractions {
val result = AspectRatio.asFraction("1260", "840")
assertThat(result).isEqualTo("3/2")
}
}
\ No newline at end of file
......@@ -11,12 +11,11 @@ import java.nio.charset.Charset
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Disabled
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertAll
import org.memobase.helpers.Constants.SettingsProps
import org.memobase.helpers.ElasticSearchWrapper
import org.memobase.helpers.JsonUtility
import org.memobase.model.FacetContainer
......@@ -103,55 +102,109 @@ class TestRecordSetSearchDoc {
}
@Test
@Disabled
fun `integration test record sets`() {
fun `kafka topology test for institution count change`() {
val settings = App.createSettings("kafkaTest1.yml")
every { elasticSearchWrapperMocked.countTotalNumberOfDocuments("") } returns 100
every { elasticSearchWrapperMocked.countNumberOfDocumentsPublished("") } returns 90
every { elasticSearchWrapperMocked.getDocumentTypesFromRecords("", "") } returns listOf(
every { elasticSearchWrapperMocked.countTotalNumberOfDocuments("testComplete") } returns 100
every { elasticSearchWrapperMocked.countNumberOfDocumentsPublished("testComplete") } returns 90
every {
elasticSearchWrapperMocked.getDocumentTypesFromRecords(
"testComplete",
"recordSet.facet"
)
} returns listOf(
FacetContainer(
LanguageContainer(listOf("Foto"), listOf("Foto"), listOf("Foto"), emptyList()), "Foto", emptyList()
)
)
every { elasticSearchWrapperMocked.getInstitutionName("completeInstitution") } returns FacetContainer(
LanguageContainer.placeholder("Complete Institution"), "completeInstitution", emptyList()
)
every { elasticSearchWrapperMocked.getInstitutionName("holdInstitution1") } returns FacetContainer(
LanguageContainer.placeholder("Hold Institution 1"), "holdInstitution1", emptyList()
)
every { elasticSearchWrapperMocked.getInstitutionName("holdInstitution2") } returns FacetContainer(
LanguageContainer.placeholder("Hold Institution 2"), "holdInstitution2", emptyList()
)
every {
elasticSearchWrapperMocked.getRecordSetName("testComplete")
} returns LanguageContainer.placeholder(
"Old Name"
)
every {
elasticSearchWrapperMocked.getExtraInstitutionsFromRecordSet("testComplete", "masterInstitution")
} returns listOf(
FacetContainer.placeholder(
"Complete Institution"
)
)
every {
elasticSearchWrapperMocked.getExtraInstitutionsFromRecordSet("testComplete", "originalInstitution")
} returns listOf(
FacetContainer.placeholder(
"Complete Institution"
)
)
every {
elasticSearchWrapperMocked.getExtraInstitutionsFromRecordSet("testComplete", "accessInstitution")
} returns listOf(
FacetContainer.placeholder(
"Complete Institution"
)
)
every {
elasticSearchWrapperMocked.getExtraInstitutionsFromRecordSet("testComplete", "institution")
} returns listOf(
FacetContainer(
LanguageContainer.placeholder("Hold Institution 1"), "holdInstitution1", emptyList()
)
)
val topology =
KafkaTopology(settings, TestUtilities.translationMappers, elasticSearchWrapperMocked)
val testDriver = TopologyTestDriver(topology.build(), settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
)
testDriver.pipeInput(
factory.create(
settings.inputTopic,
"testComplete",
readFile("completeExample.json")
)
val input = testDriver.createInputTopic(settings.inputTopic, StringSerializer(), StringSerializer())
input.pipeInput(
"testComplete",
readFile("testAddingInstitutionUpdate.json")
)
val record = testDriver.readOutput(
val record = testDriver.createOutputTopic(
settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
val report = testDriver.readOutput(
settings.processReportTopic,
val report = testDriver.createOutputTopic(
settings.processReportTopic, StringDeserializer(), StringDeserializer()
)
val updateTopicOutput = testDriver.createOutputTopic(
settings.appSettings.getProperty(SettingsProps.updateTopic),
StringDeserializer(),
StringDeserializer()
)
val reportKey = report.key()
val reportValue = reader.readValue(report.value(), Report::class.java)
val reportKeyValue = report.readKeyValue()
val reportKey = reportKeyValue.key
val reportValue = reader.readValue(reportKeyValue.value, Report::class.java)
val keyValue = record.readKeyValue()
val key = keyValue.key
val value = keyValue.value.replace(TestUtilities.dateRegex, "2020")
val resultValue = readFile("testAddingInstitutionUpdateOutput.json").replace(TestUtilities.dateRegex, "2020")
val updates = updateTopicOutput.readKeyValuesToList()
val key = record.key()
val value = record.value().replace(TestUtilities.dateRegex, "2020")
val resultValue = readFile("completeExampleOutput.json").replace(TestUtilities.dateRegex, "2020")
assertAll("",
{
assertThat(value)
.isEqualTo(resultValue)
},
{ assertThat(updates).hasSize(5) },
{ assertThat(key).isEqualTo("testComplete") },
{ assertThat(reportKey).isEqualTo("testComplete") },
{
......@@ -160,7 +213,7 @@ class TestRecordSetSearchDoc {
"testComplete",
ReportStatus.success,
"",
Service.nameRecords
Service.nameRecordSets
)
)
}
......
{
"@graph": [
{
"@id": "_:b9",
"@type": "https://www.ica.org/standards/RiC/ontology#Title",
"title": [
{
"@language": "de",
"@value": "Complete Record Set (DE)"
},
{
"@language": "fr",
"@value": "Complete Record Set (FR)"
},
{
"@language": "it",
"@value": "Complete Record Set (IT)"
}
],
"type": "main"
},
{
"@id": "_:b10",
"@type": "https://www.ica.org/standards/RiC/ontology#RecordResourceHoldingRelation",
"recordResourceHoldingRelationHasSource": "https://memobase.ch/institution/completeInstitution",
"recordResourceHoldingRelationHasTarget": "https://memobase.ch/recordSet/testComplete",
"type": "original"
},
{
"@id": "_:b21",
"@type": "https://www.ica.org/standards/RiC/ontology#Identifier",
"identifier": "testUpdate",
"type": "main"
},
{
"@id": "_:b7",
"@type": "https://www.ica.org/standards/RiC/ontology#RecordResourceHoldingRelation",
"recordResourceHoldingRelationHasSource": "https://memobase.ch/institution/completeInstitution",
"recordResourceHoldingRelationHasTarget": "https://memobase.ch/recordSet/testComplete",
"type": "master"
},
{
"@id": "_:b8",
"@type": "https://www.ica.org/standards/RiC/ontology#RecordResourceHoldingRelation",
"recordResourceHoldingRelationHasSource": "https://memobase.ch/institution/completeInstitution",
"recordResourceHoldingRelationHasTarget": "https://memobase.ch/recordSet/testComplete",
"type": "access"
},
{
"@id": "https://memobase.ch/recordSet/testComplete",
"@type": "https://www.ica.org/standards/RiC/ontology#RecordSet",
"P60451": [
"_:b4",
"https://memobase.ch/institution/mrv",
"_:b14",
"_:b15"
],
"isPublished": false,
"heldBy": ["https://memobase.ch/institution/holdInstitution1"
,"https://memobase.ch/institution/holdInstitution2"
],
"identifiedBy": [
"_:b21"
],
"recordResourceOrInstantiationIsTargetOfRecordResourceHoldingRelation": [
"_:b7",
"_:b8",
"_:b10"
],
"title": [
{
"@language": "de",
"@value": "New Name (DE)"
},
{
"@language": "it",
"@value": "New Name (IT)"
},
{
"@language": "fr",
"@value": "New Name (FR)"
}
]
}
],
"@context": {
"normalizedDateValue": {
"@id": "https://www.ica.org/standards/RiC/ontology#normalizedDateValue"
},
"conditionsOfUse": {
"@id": "https://www.ica.org/standards/RiC/ontology#conditionsOfUse"
},
"hasTitle": {
"@id": "https://www.ica.org/standards/RiC/ontology#hasTitle",
"@type": "@id"
},
"integrity": {
"@id": "https://www.ica.org/standards/RiC/ontology#integrity"
},
"conditionsOfAccess": {
"@id": "https://www.ica.org/standards/RiC/ontology#conditionsOfAccess"
},
"isRecordResourceAssociatedWithRecordResource": {
"@id": "https://www.ica.org/standards/RiC/ontology#isRecordResourceAssociatedWithRecordResource",
"@type": "@id"
},
"descriptiveNote": {
"@id": "https://www.ica.org/standards/RiC/ontology#descriptiveNote"
},
"P60496": {
"@id": "http://rdaregistry.info/Elements/u/P60496"
},
"history": {
"@id": "https://www.ica.org/standards/RiC/ontology#history"
},
"scopeAndContent": {
"@id": "https://www.ica.org/standards/RiC/ontology#scopeAndContent"
},
"conformsTo": {
"@id": "http://purl.org/dc/terms/conformsTo"
},
"identifiedBy": {
"@id": "https://www.ica.org/standards/RiC/ontology#identifiedBy",
"@type": "@id"
},
"P18": {
"@id": "http://www.wikidata.org/prop/direct/P18"
},
"isSubjectOf": {
"@id": "https://www.ica.org/standards/RiC/ontology#isSubjectOf",
"@type": "@id"
},
"P60451": {
"@id": "http://rdaregistry.info/Elements/u/P60451",
"@type": "@id"
},
"recordResourceExtent": {
"@id": "https://www.ica.org/standards/RiC/ontology#recordResourceExtent"
},
"recordResourceOrInstantiationIsTargetOfRecordResourceHoldingRelation": {
"@id": "https://www.ica.org/standards/RiC/ontology#recordResourceOrInstantiationIsTargetOfRecordResourceHoldingRelation",
"@type": "@id"
},
"title": {
"@id": "https://www.ica.org/standards/RiC/ontology#title"
},
"modificationDate": {
"@id": "https://www.ica.org/standards/RiC/ontology#modificationDate",
"@type": "http://www.w3.org/2001/XMLSchema#dateTime"
},
"hasSource": {
"@id": "https://www.ica.org/standards/RiC/ontology#hasSource",
"@type": "@id"
},
"isAssociatedWithDate": {
"@id": "https://www.ica.org/standards/RiC/ontology#isAssociatedWithDate",
"@type": "@id"
},
"publicationDate": {
"@id": "https://www.ica.org/standards/RiC/ontology#publicationDate",
"@type": "http://www.w3.org/2001/XMLSchema#date"
},
"hasLanguage": {
"@id": "https://www.ica.org/standards/RiC/ontology#hasLanguage",
"@type": "@id"
},
"isPublished": {
"@id": "https://memobase.ch/internal/isPublished",
"@type": "http://www.w3.org/2001/XMLSchema#boolean"
},
"P60099": {
"@id": "http://rdaregistry.info/Elements/u/P60099"
},
"heldBy": {
"@id": "https://www.ica.org/standards/RiC/ontology#heldBy"
},
"type": {
"@id": "https://www.ica.org/standards/RiC/ontology#type"
},
"identifier": {
"@id": "https://www.ica.org/standards/RiC/ontology#identifier"
},
"sameAs": {
"@id": "http://schema.org/sameAs"
},
"recordResourceHoldingRelationHasTarget": {
"@id": "https://www.ica.org/standards/RiC/ontology#recordResourceHoldingRelationHasTarget",
"@type": "@id"
},
"recordResourceHoldingRelationHasSource": {
"@id": "https://www.ica.org/standards/RiC/ontology#recordResourceHoldingRelationHasSource"
},
"eventType": {
"@id": "https://memobase.ch/internal/eventType"
},
"name": {
"@id": "https://www.ica.org/standards/RiC/ontology#name"
}
}
}
\ No newline at end of file
{"published":false,"name":{"de":["New Name (DE)"],"fr":["New Name (FR)"],"it":["New Name (IT)"],"un":[]},"keyVisualLink":"NoKeyVisualLinkDefined","numberOfDocuments":100,"numberOfDocumentsPublished":90,"teaserTest":{"de":[""],"fr":[""],"it":[""],"un":[""]},"documentType":[{"name":{"de":["Foto"],"fr":["Foto"],"it":["Foto"],"un":[]},"filter":"Foto"}],"supportedByMemoriav":true,"institution":[{"name":{"de":["Hold Institution 1"],"fr":["Hold Institution 1"],"it":["Hold Institution 1"],"un":["Hold Institution 1"]},"filter":"holdInstitution1"},{"name":{"de":["Hold Institution 2"],"fr":["Hold Institution 2"],"it":["Hold Institution 2"],"un":["Hold Institution 2"]},"filter":"holdInstitution2"}],"lastUpdatedDate":"2020","scopeAndContent":{"de":[""],"fr":[""],"it":[""],"un":[""]},"accessMemobase":{"de":[""],"fr":[""],"it":[""],"un":[""]},"context":{"de":[""],"fr":[""],"it":[""],"un":[""]},"originalTitle":{"de":[],"fr":[],"it":[],"un":[]},"extent":{"de":[""],"fr":[""],"it":[""],"un":[""]},"selection":{"de":[""],"fr":[""],"it":[""],"un":[""]},"indexing":{"de":[""],"fr":[""],"it":[""],"un":[""]},"rights":{"de":[""],"fr":[""],"it":[""],"un":[""]},"description":{"de":[""],"fr":[""],"it":[""],"un":[""]},"access":{"de":[""],"fr":[""],"it":[""],"un":[""]},"project":{"de":[],"fr":[],"it":[],"un":[]},"relatedRecordSets":{"de":[],"fr":[],"it":[],"un":[]},"relatedPublications":{"de":[],"fr":[],"it":[],"un":[]},"relatedDocuments":{"de":[],"fr":[],"it":[],"un":[]},"dataImport":{"de":[""],"fr":[""],"it":[""],"un":[""]},"originalInstitution":[{"name":{"de":["Complete Institution"],"fr":["Complete Institution"],"it":["Complete Institution"],"un":["Complete Institution"]},"filter":"completeInstitution"}],"masterInstitution":[{"name":{"de":["Complete Institution"],"fr":["Complete Institution"],"it":["Complete Institution"],"un":["Complete Institution"]},"filter":"completeInstitution"}],"accessInstitution":[{"name":{"de":["Complete Institution"],"fr":["Complete Institution"],"it":["Complete Institution"],"un":["Complete Institution"]},"filter":"completeInstitution"}],"id":"testComplete"}
\ No newline at end of file
......@@ -24,7 +24,7 @@
</Console>
</Appenders>
<Loggers>
<Root level="warn">
<Root level="info">
<AppenderRef ref="STDOUT"/>
</Root>
</Loggers>
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment