merge with new-cluster

.gitlab-ci.yml

@@ -4,6 +4,7 @@ stages:
   - publish
 
 variables:
+  SBT_OPTS: "-Dsbt.global.base=sbt-cache/sbtboot -Dsbt.boot.directory=sbt-cache/boot -Dsbt.ivy.home=sbt-cache/ivy"
   DOCKER_TLS_CERTDIR: ""
 
 include:
@@ -11,4 +12,13 @@ include:
     file: 'sbt-build/sbt-build.yml'
   - project: 'memoriav/memobase-2020/utilities/ci-templates'
     file: 'docker-image/docker-image.yml'
-  
\ No newline at end of file
+  - project: 'memoriav/memobase-2020/utilities/ci-templates'
+    file: 'helm-chart/helm-chart.yml'
+
+cache:
+  key: "$CI_BUILD_REF_NAME"
+  paths:
+    - "sbt-cache/ivy/cache"
+    - "sbt-cache/boot"
+    - "sbt-cache/sbtboot"
+    - "sbt-cache/target"

build.sbt

@@ -16,15 +16,15 @@ lazy val root = (project in file("."))
     test in assembly := {},
     mainClass in assembly := Some("ch.memobase.App"),
     assemblyMergeStrategy in assembly := {
-      case "log4j.properties" => MergeStrategy.first
-      case "module-info.class" => MergeStrategy.discard
-      case "log4j2.xml" => MergeStrategy.first
+      case "log4j.properties"                           => MergeStrategy.first
+      case other if other.contains("module-info.class") => MergeStrategy.discard
+      case "log4j2.xml"                                 => MergeStrategy.first
       case x =>
         val oldStrategy = (assemblyMergeStrategy in assembly).value
         oldStrategy(x)
     },
     resolvers ++= Seq(
-      "Memobase Utils" at "https://dl.bintray.com/memoriav/memobase"
+      "Memobase Libraries" at "https://gitlab.switch.ch/api/v4/projects/1324/packages/maven"
     ),
     libraryDependencies ++= Seq(
       iiifApis,
@@ -34,7 +34,11 @@ lazy val root = (project in file("."))
       log4jCore,
       log4jScala,
       log4jSlf4j,
-      memobaseServiceUtils,
+      memobaseServiceUtils excludeAll (
+        ExclusionRule(organization = "org.slf4j", name = "slf4j-api"),
+        ExclusionRule(organization = "org.slf4j", name = "slf4j-log4j12"),
+        ExclusionRule(organization = "org.slf4j", name = "jcl-over-slf4j"),
+      ),
       upickle,
       kafkaStreamsTestUtils % Test,
       scalatic % Test,

helm-charts/.helmignore

+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
\ No newline at end of file

helm-charts/Chart.yaml

+apiVersion: v2
+name: iiif-manifest-creator
+description: A helm chart for the IIIF Manifest Creator
+type: application
+version: 0.0.0
+appVersion: 0.0.0
+
+
+
+maintainers:
+  - name: Sebastian Schüpbach
+    email: sebastian.schuepbach@unibas.ch

helm-charts/helm-values/di-iiif-manifest-creator-prod.yaml

+tag: "latest"
+
+k8sEnvironment: prod
+k8sReplicas: 1
+k8sRequestsCpu: "0.2"
+k8sRequestsMemory: "128Mi"
+k8sLimitsCpu: "0.4"
+k8sLimitsMemory: "256Mi"
+
+kafkaConfigs: prod-kafka-bootstrap-servers
+inputTopicName: mb-di-processed-records-prod
+outputTopicName: mb-di-iiif-manifests-v2-prod
+reportingTopicName: mb-di-reporting-prod
+applicationId: prod-iiif-manifest-creator
+groupId: prod-iiif-manifest-creator
\ No newline at end of file

helm-charts/helm-values/di-iiif-manifest-creator-stage.yaml

+tag: "latest"
+
+k8sEnvironment: stage
+k8sReplicas: 1
+k8sRequestsCpu: "0.2"
+k8sRequestsMemory: "128Mi"
+k8sLimitsCpu: "0.4"
+k8sLimitsMemory: "256Mi"
+
+kafkaConfigs: prod-kafka-bootstrap-servers
+inputTopicName: mb-di-processed-records-stage
+outputTopicName: mb-di-iiif-manifests-v2-stage
+reportingTopicName: mb-di-reporting-stage
+applicationId: stage-iiif-manifest-creator
+groupId: stage-iiif-manifest-creator
\ No newline at end of file

helm-charts/helm-values/di-iiif-manifest-creator-test.yaml

+tag: "latest"
+
+k8sEnvironment: test
+k8sReplicas: 1
+k8sRequestsCpu: "0.2"
+k8sRequestsMemory: "128Mi"
+k8sLimitsCpu: "0.4"
+k8sLimitsMemory: "256Mi"
+
+kafkaConfigs: test-kafka-bootstrap-servers
+inputTopicName: mb-di-processed-records-prod
+outputTopicName: mb-di-iiif-manifests-v2-prod
+reportingTopicName: mb-di-reporting-prod
+applicationId: test-iiif-manifest-creator
+groupId: test-iiif-manifest-creator
\ No newline at end of file

helm-charts/templates/app-config.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+  namespace: "{{ .Values.k8sNamespace }}"
+data:
+  APPLICATION_ID: "{{ .Values.applicationId}}"
+  TOPIC_IN: "{{ .Values.inputTopicName }}"
+  TOPIC_OUT: "{{ .Values.outputTopicName }}"
+  TOPIC_PROCESS: "{{ .Values.reportingTopicName }}"

helm-charts/templates/deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}"
+  namespace: "{{ .Values.k8sNamespace }}"
+  labels:
+    app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+spec:
+  selector:
+    matchLabels:
+      app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+  replicas: {{ .Values.k8sReplicas}}
+  template:
+    metadata:
+      labels:
+        app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+        group: "{{ .Values.k8sGroupName }}"
+        environment: "{{ .Values.k8sEnvironment }}"
+    spec:
+      containers:
+        - name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-container"
+          image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
+          imagePullPolicy: Always
+          command: ["java"]
+          args: ["-Xms128m", "-Xmx256m", "-jar", "/app/app.jar"]
+          resources:
+            requests:
+              cpu: "{{ .Values.k8sRequestsCpu }}"
+              memory: "{{ .Values.k8sRequestsMemory }}"
+            limits:
+              cpu: "{{ .Values.k8sLimitsCpu }}"
+              memory: "{{ .Values.k8sLimitsMemory }}"
+          envFrom:
+            - configMapRef:
+                name: "{{ .Values.kafkaConfigs }}"
+            - configMapRef:
+                name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+      restartPolicy: Always

helm-charts/values.yaml

+registry: "cr.gitlab.switch.ch"
+image: "memoriav/memobase-2020/services/postprocessing/iiif-manifest-creator"
+tag: placeholder
+
+k8sName: iiif-manifest-creator
+k8sNamespace: memobase
+k8sGroupId: di
+k8sGroupName: documents-import
+k8sEnvironment: placeholder
+k8sReplicas: placeholder
+k8sRequestsCpu: placeholder
+k8sRequestsMemory: placeholder
+k8sLimitsCpu: placeholder
+k8sLimitsMemory: placeholder
+
+kafkaConfigs: placeholder
+inputTopicName: placeholder
+outputTopicName: placeholder
+reportingTopicName: placeholder
+applicationId: placeholder
+groupId: placeholder
\ No newline at end of file

k8s-manifests/test/deployment.yml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pp-iiif-manifest-creator-deployment
-  namespace: memobase
-  labels:
-    app: pp-iiif-manifest-creator-app
-spec:
-  selector:
-    matchLabels:
-      app: pp-iiif-manifest-creator-app
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: pp-iiif-manifest-creator-app
-        tier: web
-    spec:
-      containers:
-        - name: pp-iiif-manifest-creator-container
-          image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/postprocessing/iiif-manifest-creator:latest
-          imagePullPolicy: Always
-          command: [ "java" ]
-          args: [ "-Xmx128m", "-Xmx256m", "-jar", "/app/app.jar" ]
-          resources:
-            requests:
-              cpu: "0.2"
-              memory: "128Mi"
-            limits:
-              cpu: "0.4"
-              memory: "256Mi"
-          envFrom:
-            - configMapRef:
-                name: "prod-kafka-bootstrap-servers"
-          env:
-            - name: JOB_ID
-              value: iiif-manifest-creator
-            - name: APPLICATION_ID
-              value: iiif-manifest-creator
-            - name: TOPIC_IN
-              value: fedora-output-json-records
-            - name: TOPIC_OUT
-              value: iiif-manifests-v2
-            - name: TOPIC_PROCESS
-              value: import-process-reporting
-      restartPolicy: Always

project/Dependencies.scala

@@ -19,7 +19,7 @@
 import sbt._
 
 object Dependencies {
-  lazy val kafkaV = "2.3.1"
+  lazy val kafkaV = "2.7.0"
   lazy val log4jV = "2.11.2"
   lazy val scalatestV = "3.1.2"
 
@@ -34,7 +34,7 @@ object Dependencies {
   lazy val log4jScala = "org.apache.logging.log4j" %% "log4j-api-scala" % "11.0"
   lazy val log4jSlf4j = "org.apache.logging.log4j" % "log4j-slf4j-impl" % log4jV
   lazy val memobaseServiceUtils =
-    "org.memobase" % "memobase-service-utilities" % "1.4.1"
+    "org.memobase" % "memobase-service-utilities" % "3.0.1"
   lazy val scalatic = "org.scalactic" %% "scalactic" % scalatestV
   lazy val scalaTest = "org.scalatest" %% "scalatest" % scalatestV
   lazy val upickle = "com.lihaoyi" %% "upickle" % "0.9.5"

src/main/scala/ch/memobase/App.scala

@@ -22,7 +22,7 @@ import java.time.Duration
 
 import org.apache.kafka.streams.KafkaStreams
 import org.apache.logging.log4j.scala.Logging
-import org.memobase.settings.SettingsLoader
+import ch.memobase.settings.SettingsLoader
 
 import scala.collection.JavaConverters._
 import scala.util.{Failure, Success, Try}