Commit 6945f38c authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Update record set & institution search doc structures.

parent 893d8d49
......@@ -21,6 +21,7 @@ package org.memobase
import com.beust.klaxon.JsonArray
import com.beust.klaxon.JsonObject
import org.apache.logging.log4j.LogManager
import org.memobase.helpers.Date
import org.memobase.helpers.Extract
import org.memobase.helpers.InstitutionTypeMapper
import org.memobase.helpers.KEYS
......@@ -112,7 +113,8 @@ class InstitutionSearchDocBuilder(path: String) {
}
else
it as String
}
},
lastUpdatedDate = Date.now
)
}
......
......@@ -42,6 +42,8 @@ class KafkaTopology(private val settings: SettingsLoader) {
private val searchDocTransform = SearchDocTransform(settings.appSettings.getProperty(KEYS.SettingsProps.mediaUrl))
private val institutionSearchDoc = InstitutionSearchDocBuilder(settings.appSettings.getProperty(KEYS.SettingsProps.institutionTypeLabelsPath))
private val jsonWriter = ObjectMapper().registerKotlinModule().writer()
fun build(): Topology {
val builder = StreamsBuilder()
val stream = builder.stream<String, String>(settings.inputTopic)
......@@ -76,9 +78,6 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
outputStreams(institutionStream)
return builder.build()
}
......@@ -91,9 +90,9 @@ class KafkaTopology(private val settings: SettingsLoader) {
.filterNot { _, value -> value.second.status == "FAILURE" }
.mapValues { value -> value.first }
.mapValues { value ->
val writer = StringWriter()
ObjectMapper().registerKotlinModule().writeValue(writer, value)
writer.toString()
val out = StringWriter()
jsonWriter.writeValue(out, value)
out.toString()
}
.to(settings.outputTopic)
}
......
package org.memobase.helpers
import java.time.LocalDateTime
import java.time.format.DateTimeFormatter
object Date {
val now = LocalDateTime.now().format(DateTimeFormatter.ISO_DATE_TIME)
}
\ No newline at end of file
......@@ -16,6 +16,7 @@ object Default {
emptyList(),
emptyList(),
"",
"",
0,
0,
"",
......
......@@ -24,7 +24,7 @@ import com.fasterxml.jackson.annotation.JsonInclude
data class FacetContainer(
val name: LanguageContainer,
val filter: String?,
val facet: List<String>
val facet: List<String>?
) {
companion object {
val DEFAULT = FacetContainer(
......
......@@ -21,6 +21,9 @@ data class InstitutionSearchDoc(
val type: List<FacetContainer>,
val documentType: List<FacetContainer>,
// Sort
val lastUpdatedDate: String,
// Search Result Display
val keyVisualLink: String,
val numberOfRecordSets: Int,
......
......@@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonInclude
import org.memobase.helpers.KEYS
@JsonInclude(JsonInclude.Include.NON_NULL)
data class LanguageContainer(
open class LanguageContainer(
val de: List<String>,
val fr: List<String>,
val it: List<String>,
......
......@@ -18,8 +18,14 @@ data class RecordSetSearchDoc(
val institution: FacetContainer,
val periodOfTimeAsYear: String,
// Sort
val lastUpdatedDate: String,
// search
val scopeAndContent: LanguageContainer
val scopeAndContent: LanguageContainer,
// Display
val keyVisualLink: String
) : Schema(recordSetId) {
companion object {
......@@ -41,11 +47,13 @@ data class RecordSetSearchDoc(
),
institution = FacetContainer(
LanguageContainer.placeholder("TEST INSTITUTION"),
filter = "",
filter = "INSTITUTION_IDENTIFIER",
facet = emptyList()
),
periodOfTimeAsYear = "2020",
scopeAndContent = LanguageContainer.placeholder("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.")
scopeAndContent = LanguageContainer.placeholder("Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."),
lastUpdatedDate = "2020-11-20T10:29:01.128",
keyVisualLink = "https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2020-10/1.jpg?itok=5ncVBnVQ"
)
}
}
\ No newline at end of file
......@@ -44,8 +44,6 @@ class IntegrationTest {
return File("$resourcePath/$fileName").readText(Charset.defaultCharset())
}
private val klaxon = Klaxon()
@ParameterizedTest
@MethodSource("testParams")
fun `integration tests`(params: TestParam) {
......@@ -90,56 +88,4 @@ class IntegrationTest {
"key"
)
)
@Test
fun `integration test institution`() {
val service = Service("kafkaTest1.yml")
val testDriver = TopologyTestDriver(service.topology, service.settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
)
testDriver.pipeInput(
factory.create(
service.settings.inputTopic,
"https://memobase.ch/institution/RadioX",
readFile("institution/input.json")
)
)
val record = testDriver.readOutput(
service.settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
val report = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
val reportKey = report.key()
val reportValue = klaxon.parse<Report>(report.value())!!
val value = record.value()
val key = record.key()
assertAll("",
{
assertThat(value)
.isEqualTo(readFile("institution/output.json"))
},
{ assertThat(key).isEqualTo("https://memobase.ch/institution/RadioX") },
{ assertThat(reportKey).isEqualTo("https://memobase.ch/institution/RadioX") },
{
assertThat(reportValue).isEqualTo(
Report(
"https://memobase.ch/institution/RadioX",
"SUCCESS",
"Transformed message into search doc."
)
)
}
)
}
}
package org.memobase
import com.beust.klaxon.Klaxon
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.module.kotlin.registerKotlinModule
import org.apache.kafka.common.serialization.StringDeserializer
import org.apache.kafka.common.serialization.StringSerializer
import org.apache.kafka.streams.TopologyTestDriver
import org.apache.kafka.streams.test.ConsumerRecordFactory
import org.assertj.core.api.Assertions.assertThat
import org.junit.jupiter.api.Test
import org.junit.jupiter.api.TestInstance
import org.junit.jupiter.api.assertAll
import org.memobase.model.InstitutionSearchDoc
import org.memobase.model.Report
import java.io.File
import java.io.StringReader
import java.io.StringWriter
import java.nio.charset.Charset
import kotlin.reflect.typeOf
@TestInstance(TestInstance.Lifecycle.PER_CLASS)
class TestInstitutionSearchDoc {
private val reader = ObjectMapper().registerKotlinModule().reader()
private val dataPath = "src/test/resources/data/institution"
private fun readFile(fileName: String): String {
return File("$dataPath/$fileName").readText(Charset.defaultCharset())
}
private val dateRegex = Regex("\\d{4}-\\d{2}-\\d{2}T\\d{2}:\\d{2}:\\d{2}.\\d{3}")
@Test
fun `integration test institution`() {
val service = Service("kafkaTest1.yml")
val testDriver = TopologyTestDriver(service.topology, service.settings.kafkaStreamsSettings)
val factory = ConsumerRecordFactory(
StringSerializer(), StringSerializer()
)
testDriver.pipeInput(
factory.create(
service.settings.inputTopic,
"https://memobase.ch/institution/RadioX",
readFile("input.json")
)
)
val record = testDriver.readOutput(
service.settings.outputTopic,
StringDeserializer(),
StringDeserializer()
)
val report = testDriver.readOutput(
service.settings.processReportTopic,
StringDeserializer(),
StringDeserializer()
)
val reportKey = report.key()
val reportValue = reader.readValue(report.value(), Report::class.java)
val key = record.key()
val value = record.value().replace(dateRegex, "2020")
val resultValue = readFile("output.json").replace(dateRegex, "2020")
assertAll("",
{
assertThat(value)
.isEqualTo(resultValue)
},
{ assertThat(key).isEqualTo("https://memobase.ch/institution/RadioX") },
{ assertThat(reportKey).isEqualTo("https://memobase.ch/institution/RadioX") },
{
assertThat(reportValue).isEqualTo(
Report(
"https://memobase.ch/institution/RadioX",
"SUCCESS",
"Transformed message into search doc."
)
)
}
)
}
}
\ No newline at end of file
{"published":false,"name":{"de":["RadioX"],"fr":["Test institution Oana (FR)"],"it":["Test institution Oana (IT)"],"un":[]},"description":{"de":["<p>Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem.</p>"],"fr":["<p>test_description&nbsp;(FR)</p>"],"it":["<p>test_description&nbsp;&nbsp;(IT)</p>"],"un":[]},"city":[{"de":["Münchenstein"],"fr":["Münchenstein"],"it":["Münchenstein"],"un":[]},{"de":[],"fr":[],"it":[],"un":["Bern"]}],"address":["Helvetiapl. 5","Oslo-Strasse 8"],"postalCodes":["3005","4142"],"canton":[{"name":{"de":["Bern"],"fr":["Berne"],"it":["Berne"],"un":[]}},{"name":{"de":["Basel-Stadt"],"fr":["Bâle-Ville"],"it":["Bâle-Ville"],"un":[]}}],"type":[{"name":{"de":["Behörde"],"fr":["autorité"],"it":["autorità"],"un":[]}},{"name":{"de":["Archiv"],"fr":["Archives"],"it":["Archivio"],"un":[]}}],"documentType":[{"name":{"de":["PLACEHOLDER"],"fr":["PLACEHOLDER"],"it":["PLACEHOLDER"],"un":["PLACEHOLDER"]},"filter":"PLACEHOLDER","facet":["PLACEHOLDER"]}],"keyVisualLink":"https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2020-10/1.jpg?itok=5ncVBnVQ","numberOfRecordSets":1,"numberOfDocuments":0,"id":"RadioX"}
\ No newline at end of file
{"published":false,"name":{"de":["RadioX"],"fr":["Test institution Oana (FR)"],"it":["Test institution Oana (IT)"],"un":[]},"description":{"de":["<p>Sed ut perspiciatis unde omnis iste natus error sit voluptatem accusantium doloremque laudantium, totam rem aperiam, eaque ipsa quae ab illo inventore veritatis et quasi architecto beatae vitae dicta sunt explicabo. Nemo enim ipsam voluptatem quia voluptas sit aspernatur aut odit aut fugit, sed quia consequuntur magni dolores eos qui ratione voluptatem sequi nesciunt. Neque porro quisquam est, qui dolorem ipsum quia dolor sit amet, consectetur, adipisci velit, sed quia non numquam eius modi tempora incidunt ut labore et dolore magnam aliquam quaerat voluptatem.</p>"],"fr":["<p>test_description&nbsp;(FR)</p>"],"it":["<p>test_description&nbsp;&nbsp;(IT)</p>"],"un":[]},"city":[{"de":["Münchenstein"],"fr":["Münchenstein"],"it":["Münchenstein"],"un":[]},{"de":[],"fr":[],"it":[],"un":["Bern"]}],"address":["Helvetiapl. 5","Oslo-Strasse 8"],"postalCodes":["3005","4142"],"canton":[{"name":{"de":["Bern"],"fr":["Berne"],"it":["Berne"],"un":[]}},{"name":{"de":["Basel-Stadt"],"fr":["Bâle-Ville"],"it":["Bâle-Ville"],"un":[]}}],"type":[{"name":{"de":["Behörde"],"fr":["autorité"],"it":["autorità"],"un":[]}},{"name":{"de":["Archiv"],"fr":["Archives"],"it":["Archivio"],"un":[]}}],"documentType":[{"name":{"de":["PLACEHOLDER"],"fr":["PLACEHOLDER"],"it":["PLACEHOLDER"],"un":["PLACEHOLDER"]},"filter":"PLACEHOLDER","facet":["PLACEHOLDER"]}],"lastUpdatedDate":"2020-11-20T10:29:59.141","keyVisualLink":"https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2020-10/1.jpg?itok=5ncVBnVQ","numberOfRecordSets":1,"numberOfDocuments":0,"id":"RadioX"}
\ No newline at end of file
{"name":{"de":["TEST_RECORD_SET"],"fr":["TEST_RECORD_SET"],"it":["TEST_RECORD_SET"],"un":["TEST_RECORD_SET"]},"isPublished":false,"documentType":{"name":{"de":["TEST DOCUMENT TYPE"],"fr":["TEST DOCUMENT TYPE"],"it":["TEST DOCUMENT TYPE"],"un":["TEST DOCUMENT TYPE"]}},"supportedByMemoriav":true,"languageOfMetadata":{"name":{"de":["TEST LANGUAGE"],"fr":["TEST LANGUAGE"],"it":["TEST LANGUAGE"],"un":["TEST LANGUAGE"]}},"institution":{"name":{"de":["TEST INSTITUTION"],"fr":["TEST INSTITUTION"],"it":["TEST INSTITUTION"],"un":["TEST INSTITUTION"]}},"periodOfTimeAsYear":"2020","scopeAndContent":{"de":["Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."],"fr":["Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."],"it":["Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."],"un":["Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."]},"id":"NoRecordSetId"}
\ No newline at end of file
{"name":{"de":["TEST_RECORD_SET"],"fr":["TEST_RECORD_SET"],"it":["TEST_RECORD_SET"],"un":["TEST_RECORD_SET"]},"isPublished":false,"documentType":{"name":{"de":["TEST DOCUMENT TYPE"],"fr":["TEST DOCUMENT TYPE"],"it":["TEST DOCUMENT TYPE"],"un":["TEST DOCUMENT TYPE"]}},"supportedByMemoriav":true,"languageOfMetadata":{"name":{"de":["TEST LANGUAGE"],"fr":["TEST LANGUAGE"],"it":["TEST LANGUAGE"],"un":["TEST LANGUAGE"]}},"institution":{"name":{"de":["TEST INSTITUTION"],"fr":["TEST INSTITUTION"],"it":["TEST INSTITUTION"],"un":["TEST INSTITUTION"]},"filter":"INSTITUTION_IDENTIFIER"},"periodOfTimeAsYear":"2020","lastUpdatedDate":"2020-11-20T10:29:01.128","scopeAndContent":{"de":["Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."],"fr":["Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."],"it":["Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."],"un":["Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum."]},"keyVisualLink":"https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2020-10/1.jpg?itok=5ncVBnVQ","id":"NoRecordSetId"}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment