move to helm

.gitlab-ci.yml

@@ -10,6 +10,8 @@ include:
   - project: 'memoriav/memobase-2020/utilities/ci-templates'
     ref: modular
     file: 'docker/base.yml'
+  - project: 'memoriav/memobase-2020/utilities/ci-templates'
+    file: 'helm-chart/helm-chart.yml'
 
 cache:
   key: "$CI_BUILD_REF_NAME"

helm-charts/.helmignore

+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
\ No newline at end of file

helm-charts/Chart.yaml

+apiVersion: v2
+name: media-converter
+description: A helm chart for the Media Converter
+type: application
+version: 0.0.0
+appVersion: 0.0.0
+
+
+
+maintainers:
+  - name: Sebastian Schüpbach
+    email: sebastian.schuepbach@unibas.ch

helm-charts/templates/app-config.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+  namespace: "{{ .Values.k8sNamespace }}"
+data:
+  APPLICATION_ID: "{{ .Values.applicationId}}"
+  TOPIC_IN: "{{ .Values.inputTopicName }}"
+  TOPIC_PROCESS: "{{ .Values.reportingTopicName }}"
+  CLIENT_ID: "{{. Values.clientId }}"
+  GROUP_ID: "{{. Values.groupId }}"
+  AUDIO_SNIPPET_DURATION: "{{ .Values.audioSnippetDuration }}"
+  EXTERNAL_BASE_URL: "{{ .Values.externalBaseUrl }}"
+  MEDIA_FOLDER_ROOT_PATH: "{{ .Values.mediaFolderRootPath }}"
+  DISTRIBUTOR_URL: "{{ .Values.distributorUrl }}"
+  CONNECTION_RETRY_AFTER_MS: {{ .Values.connectionRetryAfterMs }}
+  CONNECTION_MAX_RETRIES: "{{ .Values.connectionMaxRetries }}"

helm-charts/templates/deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}"
+  namespace: "{{ .Values.k8sNamespace }}"
+  labels:
+    app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+spec:
+  selector:
+    matchLabels:
+      app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+  replicas: {{ .Values.k8sReplicas}}
+  template:
+    metadata:
+      labels:
+        app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+        group: "{{ .Values.k8sGroupName }}"
+        environment: "{{ .Values.k8sEnvironment }}"
+    spec:
+      containers:
+        - name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-container"
+          image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
+          imagePullPolicy: Always
+          command: [ "java" ]
+          args: [ "-Xms128m", "-Xmx10g", "-jar", "/app/app.jar" ]
+          resources:
+            requests:
+              cpu: "{{ .Values.k8sRequestsCpu }}"
+              memory: "{{ .Values.k8sRequestsMemory }}"
+            limits:
+              cpu: "{{ .Values.k8sLimitsCpu }}"
+              memory: "{{ .Values.k8sLimitsMemory }}"
+          envFrom:
+            - configMapRef:
+                name: "{{ .Values.kafkaConfigs }}"
+            - configMapRef:
+                name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+          volumeMounts:
+            - name: media-volume
+              mountPath: "{{ .Values.mediaFolderRootPath }}"
+      restartPolicy: Always
+      volumes:
+        - name: media-volume
+          persistentVolumeClaim:
+            claimName: "{{ .Values.mediaVolumeClaimName }}"

helm-charts/values-prod.yaml

+registry: "cr.gitlab.switch.ch"
+image: "memoriav/memobase-2020/services/postprocessing/media-converter"
+tag: "latest"
+
+k8sName: media-converter
+k8sNamespace: memobase
+k8sGroupId: di
+k8sGroupName: documents-import
+k8sEnvironment: prod
+k8sReplicas: 1
+k8sRequestsCpu: "0.2"
+k8sRequestsMemory: "1Gi"
+k8sLimitsCpu: "1"
+k8sLimitsMemory: "11Gi"
+
+kafkaConfigs: prod-kafka-bootstrap-servers
+# TODO: To be changed
+inputTopicName: fedora-output-json-records
+# TODO: To be changed
+reportingTopicName: import-process-reporting
+applicationId: prod-iiif-manifest-creator
+groupId: prod-media-converter
+clientId: prod-media-converter-client
+
+audioSnippetDuration: "30"
+
+externalBaseUrl: "https://memobase.ch/"
+distributorUrl: "http://mb-wf2.memobase.unibas.ch:3000"
+
+mediaFolderRootPath: "/data"
+mediaVolumeClaimName: media-volume-claim
+
+connectionRetryAfterMs: "10000"
+connectionMaxRetries: "3"

helm-charts/values-stage.yaml

+registry: "cr.gitlab.switch.ch"
+image: "memoriav/memobase-2020/services/postprocessing/media-converter"
+tag: "latest"
+
+k8sName: media-converter
+k8sNamespace: memobase
+k8sGroupId: di
+k8sGroupName: documents-import
+k8sEnvironment: stage
+k8sReplicas: 1
+k8sRequestsCpu: "0.2"
+k8sRequestsMemory: "1Gi"
+k8sLimitsCpu: "1"
+k8sLimitsMemory: "11Gi"
+
+kafkaConfigs: prod-kafka-bootstrap-servers
+# TODO: To be changed
+inputTopicName: fedora-output-json-records
+# TODO: To be changed
+reportingTopicName: import-process-reporting
+applicationId: stage-iiif-manifest-creator
+groupId: stage-media-converter
+clientId: stage-media-converter-client
+
+audioSnippetDuration: "30"
+
+externalBaseUrl: "https://memobase.ch/"
+distributorUrl: "http://mb-wf2.memobase.unibas.ch:3000"
+
+mediaFolderRootPath: "/data"
+mediaVolumeClaimName: stage-media-volume-claim
+
+connectionRetryAfterMs: "10000"
+connectionMaxRetries: "3"

helm-charts/values-test.yaml

+registry: "cr.gitlab.switch.ch"
+image: "memoriav/memobase-2020/services/postprocessing/media-converter"
+tag: "latest"
+
+k8sName: media-converter
+k8sNamespace: memobase
+k8sGroupId: di
+k8sGroupName: documents-import
+k8sEnvironment: test
+k8sReplicas: 1
+k8sRequestsCpu: "0.2"
+k8sRequestsMemory: "1Gi"
+k8sLimitsCpu: "1"
+k8sLimitsMemory: "2Gi"
+
+kafkaConfigs: test-kafka-bootstrap-servers
+# TODO: To be changed
+inputTopicName: fedora-output-json-records
+# TODO: To be changed
+reportingTopicName: import-process-reporting
+applicationId: test-iiif-manifest-creator
+groupId: test-media-converter
+clientId: test-media-converter-client
+
+audioSnippetDuration: "30"
+
+externalBaseUrl: "https://memobase.ch/"
+distributorUrl: "http://mb-wf2.memobase.unibas.ch:3000"
+
+mediaFolderRootPath: "/data"
+mediaVolumeClaimName: test-media-volume-claim
+
+connectionRetryAfterMs: "10000"
+connectionMaxRetries: "3"

k8s-manifests/test/deployment.yml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pp-media-converter-deployment
-  namespace: memobase
-  labels:
-    app: pp-media-converter-app
-spec:
-  selector:
-    matchLabels:
-      app: pp-media-converter-app
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: pp-media-converter-app
-        tier: postprocessing
-    spec:
-      containers:
-        - name: pp-media-converter-container
-          image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/postprocessing/media-converter:latest
-          imagePullPolicy: Always
-          command: [ "java" ]
-          args: [ "-Xms128m", "-Xmx10g", "-jar", "/app/app.jar" ]
-          resources:
-            requests:
-              cpu: "0.2"
-              memory: "1Gi"
-            limits:
-              cpu: "1"
-              memory: "11Gi"
-          volumeMounts:
-            - name: media-volume
-              mountPath: /data
-          envFrom:
-            - configMapRef:
-                name: "prod-kafka-bootstrap-servers"
-          env:
-            - name: JOB_ID
-              value: media-converter
-            - name: AUDIO_SNIPPET_DURATION
-              value: "30"
-            - name: EXTERNAL_BASE_URL
-              value: "https://memobase.ch/"
-            - name: CLIENT_ID
-              value: media-converter-client
-            - name: GROUP_ID
-              value: media-converter
-            - name: TOPIC_IN
-              value: fedora-output-json-records
-            - name: TOPIC_PROCESS
-              value: import-process-reporting
-            - name: MEDIA_FOLDER_ROOT_PATH
-              value: "/data"
-            - name: DISTRIBUTOR_URL
-              value: "http://mb-wf2.memobase.unibas.ch:3000"
-            - name: CONNECTION_RETRY_AFTER_MS
-              value: "10000"
-            - name: CONNECTION_MAX_RETRIES
-              value: "3"
-
-      restartPolicy: Always
-      volumes:
-        - name: media-volume
-          persistentVolumeClaim:
-            claimName: media-volume-claim