Commit 2924c676 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Make computed teaser image optional.

Expand record set entity.
Improve error handling for json parser.
Fix input type adapter.
parent 285329ef
Pipeline #17700 passed with stages
in 4 minutes and 18 seconds
......@@ -20,14 +20,15 @@ package org.memobase
import com.beust.klaxon.Klaxon
import com.beust.klaxon.KlaxonException
import java.io.StringReader
import org.apache.jena.rdf.model.Model
import org.apache.kafka.streams.KeyValue
import org.apache.kafka.streams.StreamsBuilder
import org.apache.kafka.streams.Topology
import org.apache.logging.log4j.LogManager
import org.memobase.model.*
import org.memobase.model.Input
import org.memobase.model.Institution
import org.memobase.model.RecordSet
import org.memobase.settings.SettingsLoader
import java.io.StringReader
class KafkaTopology(private val settings: SettingsLoader) {
private val log = LogManager.getLogger("DrupalSyncProcessor")
......@@ -38,7 +39,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
val stream = builder.stream<String, String>(settings.inputTopic)
stream
.flatMapValues { value -> parseJson(value) }
.mapValues { value -> transformJson(value) }
.flatMapValues { value -> transformJson(value) }
.map { _, value -> transformer.write(value.first, value.second) }
.to(settings.outputTopic)
......@@ -59,11 +60,14 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
}
private fun transformJson(input: Input): Pair<String, Model> {
private fun transformJson(input: Input): List<Pair<String, Model>> {
return when (input) {
is Institution -> transformer.createInstitution(input)
is RecordSet -> transformer.createRecordSet(input)
else -> throw Exception("Unknown Type Input.")
is Institution -> listOf(transformer.createInstitution(input))
is RecordSet -> listOf(transformer.createRecordSet(input))
else -> {
log.error("Could not cast input to institution or record set: $input.")
return emptyList()
}
}
}
}
......@@ -99,7 +99,8 @@ class Transform {
input.field_institution_types.forEach {
resource.addProperty(WD.typeOfInstitution, model.createResource(NS.wd + it.substringAfterLast("/")))
}
resource.addLiteral(WD.image, input.computed_teaser_image_url)
if (input.computed_teaser_image_url != null)
resource.addLiteral(WD.image, input.computed_teaser_image_url)
return Pair(resource.uri, model)
}
......
......@@ -23,7 +23,7 @@ import kotlin.reflect.KClass
class InputTypeAdapter : TypeAdapter<Input> {
override fun classFor(type: Any): KClass<out Input> = when(type as String) {
"node--institution" -> Institution::class
"node--recordset" -> RecordSet::class
"node--record_set" -> RecordSet::class
else -> throw IllegalArgumentException("Unknown type: $type")
}
}
\ No newline at end of file
......@@ -42,5 +42,5 @@ data class Institution(
val field_institution_types: List<String>,
val field_wikidata_id: String,
val computed_teaser_image_url: String
val computed_teaser_image_url: String?
) : Input("Institution")
\ No newline at end of file
......@@ -17,4 +17,17 @@
*/
package org.memobase.model
data class RecordSet(val title: String) : Input("RecordSet")
\ No newline at end of file
data class RecordSet(
val title: String,
val title_fr: String,
val title_it: String,
val field_text: RichText,
val field_text_fr: RichText,
val field_text_it: RichText,
val field_memobase_id: String,
val institutions_ids: List<String>,
val computed_teaser_image_url: String
) : Input("RecordSet")
\ No newline at end of file
{
"type": "node--institution",
"status": true,
"title": "Schweizer Radio und Fernsehen",
"title_fr": "Schweizer Radio und Fernsehen",
"title_it": "Schweizer Radio und Fernsehen",
"field_address": [
{
"langcode": "",
"country_code": "CH",
"administrative_area": "",
"locality": "Zürich",
"dependent_locality": null,
"postal_code": "8052",
"sorting_code": null,
"address_line1": "Fernsehstrasse 1-4",
"address_line2": "",
"organization": "",
"given_name": null,
"additional_name": null,
"family_name": null,
"coordinates": "8.53547, 47.40146"
}
],
"field_isil": "CH-000539-X",
"field_memobase_id": "srf",
"field_old_memobase_id": "SRF",
"field_email": "archiv@srf.ch",
"field_website": {
"uri": "https://www.srf.ch",
"title": "",
"options": []
},
"field_wikidata_id": "http://www.wikidata.org/wiki/Q27490292",
"field_link_archive_catalog": {
"uri": "https://www.srf.ch/play",
"title": "",
"options": []
},
"field_text": {
"value": "<p>Schweizer Radio und Fernsehen (SRF) sendet aus den drei Hauptstudios in Basel, Bern und Zürich</p>\r\n",
"format": "editorial",
"processed": "<p>Schweizer Radio und Fernsehen (SRF) sendet aus den drei Hauptstudios in Basel, Bern und Zürich</p>"
},
"field_text_fr": {
"value": "<p>Schweizer Radio und Fernsehen (SRF) sendet aus den drei Hauptstudios in Basel, Bern und Zürich</p>\r\n",
"format": "editorial",
"processed": "<p>Schweizer Radio und Fernsehen (SRF) sendet aus den drei Hauptstudios in Basel, Bern und Zürich</p>"
},
"field_text_it": {
"value": "<p>Schweizer Radio und Fernsehen (SRF) sendet aus den drei Hauptstudios in Basel, Bern und Zürich</p>\r\n",
"format": "editorial",
"processed": "<p>Schweizer Radio und Fernsehen (SRF) sendet aus den drei Hauptstudios in Basel, Bern und Zürich</p>"
},
"field_institution_types": [
"http://www.wikidata.org/wiki/Q166118",
"http://www.wikidata.org/wiki/Q15265344"
],
"recordset_ids": [],
"computed_teaser_image_url": null
}
\ No newline at end of file
......@@ -65,14 +65,14 @@ mbcb:RadioX a rico:CorporateBody ;
wdt:P669 "Oslo-Strasse" ;
wdt:P670 "8"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "RadioX" ;
rico:type "main"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "test_radio_x" ;
rico:type "oldMemobase"
] ;
rico:identifiedBy [ a rico:Identifier ;
rico:identifier "RadioX" ;
rico:type "main"
] ;
rico:isHolderOf "https://memobase.ch/recordSet/RadioX-Kampagnen" ;
rico:name "RadioX"@de , "Test institution Oana (FR)"@fr , "Test institution Oana (IT)"@it ;
rico:type "memobaseInstitution" .
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment