Commit 1c5d2a96 authored by Günter Hipler's avatar Günter Hipler
Browse files

finalize component and helm package deployment

parent 65f48478
Pipeline #31339 passed with stages
in 7 minutes and 32 seconds
# Creation of Europeana Data Model (EDM) documents
### Purpose
Transforms memobase Rico information into the EDM model
(the work is strongly inspired by earlier work by my colleagues Sebastian and Jonas- thanks!
- iiif-manifest-creator
- normalization-service
- mapper-service
- media-linker
- fedora-ingest-service
- drupal-sync-service
)
### Mapping Rules
[rules](https://ub-basel.atlassian.net/wiki/spaces/MEMOBASE/pages/270368788/Mapping+Memobase+to+EDM)
### Implementation
* Scala
* Kafka Streams
### deployment
* Helm package manager for k8s
[chart](https://gitlab.switch.ch/memoriav/memobase-2020/services/postprocessing/rico-edm-transformer/-/tree/master/helm-charts)
tag: 0.3.3
tag: "latest"
k8sEnvironment: prod
k8sReplicas: 1
k8sRequestsCpu: "0.2"
k8sRequestsMemory: "128Mi"
k8sRequestsMemory: "512Mi"
k8sLimitsCpu: "0.4"
k8sLimitsMemory: "256Mi"
k8sLimitsMemory: "2048Mi"
kafkaConfigs: prod-kafka-bootstrap-servers
inputTopicName: mb-di-processed-records-prod
outputTopicName: mb-di-edm-es-records-prod
reportingTopicName: mb-di-reporting-prod
applicationId: prod-rico-edm-transformer
applicationId: prod-rico-edm-transformer-v2
elasticHostConfigs: prod-elastic-configs
institutionIndexCM: prod-institutions-index
......
tag: "0.3.3"
tag: "latest"
k8sEnvironment: stage
......
tag: "0.3.3"
tag: "latest"
k8sEnvironment: test
......
......@@ -19,10 +19,11 @@ spec:
spec:
containers:
- name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-container"
image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
#image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
image: guenterh/rico-edm-transformer:0.3.4
imagePullPolicy: Always
command: [ "java" ]
args: [ "-Xms128m", "-Xmx256m", "-jar", "/app/app.jar" ]
args: [ "-Xms512m", "-Xmx2048m", "-jar", "/app/app.jar" ]
resources:
requests:
cpu: "{{ .Values.k8sRequestsCpu }}"
......
......@@ -21,11 +21,13 @@
package ch.memobase.rico2edm
import ch.memobase.rico2edm.utils.{ElasticSearchClientWrapper, Helper, Keys}
import org.apache.kafka.streams.KafkaStreams
import org.apache.kafka.streams.{KafkaStreams, StreamsConfig}
import org.apache.logging.log4j.scala.Logging
import ch.memobase.settings.SettingsLoader
import org.apache.kafka.streams.processor.WallclockTimestampExtractor
import java.time.Duration
import java.util.Properties
import scala.util.{Failure, Success, Try}
import scala.jdk.CollectionConverters._
......@@ -57,6 +59,9 @@ object Main extends Logging {
false
)
//use processingTime for newly created messages in topic otherwise mesages could be immediately deleted
val tempSettings: Properties = settings.getKafkaStreamsSettings
tempSettings.put(StreamsConfig.DEFAULT_TIMESTAMP_EXTRACTOR_CLASS_CONFIG, classOf[WallclockTimestampExtractor])
val streams = new KafkaStreams(
topology.build(
......@@ -66,7 +71,7 @@ object Main extends Logging {
settings.getAppSettings
),
settings.getKafkaStreamsSettings
tempSettings
)
val shutdownGracePeriodMs = 10000
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment