move to helm

Signed-off-by: Sebastian Schüpbach <sebastian.schuepbach@unibas.ch>

helm-charts/.helmignore

+# Patterns to ignore when building packages.
+# This supports shell glob matching, relative path matching, and
+# negation (prefixed with !). Only one pattern per line.
+.DS_Store
+# Common VCS dirs
+.git/
+.gitignore
+.bzr/
+.bzrignore
+.hg/
+.hgignore
+.svn/
+# Common backup files
+*.swp
+*.bak
+*.tmp
+*.orig
+*~
+# Various IDEs
+.project
+.idea/
+*.tmproj
+.vscode/
\ No newline at end of file

helm-charts/Chart.yaml

+apiVersion: v2
+name: iiif-manifest-indexer
+description: A helm chart for the Media Metadata Indexer
+type: application
+version: 0.0.0
+appVersion: 0.0.0
+
+
+
+maintainers:
+  - name: Sebastian Schüpbach
+    email: sebastian.schuepbach@unibas.ch

helm-charts/templates/app-config.yaml

+apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+  namespace: "{{ .Values.k8sNamespace }}"
+data:
+  LOG_LEVEL: "{{ .Values.logLevel }}"
+  URI_BASE: "{{ .Values.uriBase }}"
+  GROUP_ID: "{{ .Values.groupId}}"
+  TOPIC_IN: "{{ .Values.inputTopicName }}"
+  TOPIC_PROCESS: "{{ .Values.reportingTopicName }}"
+  KAFKA_CONNECTION_RETRIES: "{{ .Values.kafkaConnectionRetries }}"
+  MARIADB_CONNECTION_RETRIES: "{{ .Values.mariadbConnectionRetries }}"
\ No newline at end of file

helm-charts/templates/deployment.yaml

+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}"
+  namespace: "{{ .Values.k8sNamespace }}"
+  labels:
+    app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+spec:
+  selector:
+    matchLabels:
+      app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+  replicas: { { .Values.k8sReplicas } }
+  template:
+    metadata:
+      labels:
+        app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
+        group: "{{ .Values.k8sGroupName }}"
+        environment: "{{ .Values.k8sEnvironment }}"
+    spec:
+      containers:
+        - name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-container"
+          image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
+          imagePullPolicy: Always
+          resources:
+            requests:
+              cpu: "{{ .Values.k8sRequestsCpu }}"
+              memory: "{{ .Values.k8sRequestsMemory }}"
+            limits:
+              cpu: "{{ .Values.k8sLimitsCpu }}"
+              memory: "{{ .Values.k8sLimitsMemory }}"
+          envFrom:
+            - configMapRef:
+                name: "{{ .Values.kafkaConfigs }}"
+            - configMapRef:
+                name: "{{ .Values.mariadbDatabaseConfigs }}"
+            - configMapRef:
+                name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
+          env:
+            - name: MARIADB_PASSWORD
+              valueFrom:
+                secretKeyRef:
+                  name: { { .Values.mariadbUserSecrets } }
+                  key: MARIADB_PASSWORD
+            - name: MARIADB_USER
+              valueFrom:
+                secretKeyRef:
+                  name: { { .Values.mariadbUserSecrets } }
+                  key: MARIADB_USER
+      restartPolicy: Always

helm-charts/values-prod.yaml

+registry: "cr.gitlab.switch.ch"
+image: "memoriav/memobase-2020/services/postprocessing/mediametadatatodb"
+tag: "latest"
+
+k8sName: media-metadata-indexer
+k8sNamespace: memobase
+k8sGroupId: di
+k8sGroupName: documents-import
+k8sEnvironment: prod
+k8sReplicas: 1
+k8sRequestsCpu: "0.1"
+k8sRequestsMemory: "0.1Gi"
+k8sLimitsCpu: "0.8"
+k8sLimitsMemory: "0.8Gi"
+
+logLevel: INFO
+uriBase: "file:///data/"
+
+kafkaConfigs: prod-kafka-bootstrap-servers
+# TODO: To be changed
+inputTopicName: fedora-output-json-records
+# TODO: To be changed
+reportingTopicName: import-process-reporting
+groupId: prod-media-metadata-indexer
+kafkaConnectionRetries: 2
+
+mariadbDatabaseConfigs: prod-mariadb-db-medienserver-configs
+mariadbUserSecrets: prod-mariadb-user-medienserver-secrets
+mariadbConnectionRetries: 2

helm-charts/values-stage.yaml

+registry: "cr.gitlab.switch.ch"
+image: "memoriav/memobase-2020/services/postprocessing/mediametadatatodb"
+tag: "latest"
+
+k8sName: media-metadata-indexer
+k8sNamespace: memobase
+k8sGroupId: di
+k8sGroupName: documents-import
+k8sEnvironment: stage
+k8sReplicas: 1
+k8sRequestsCpu: "0.1"
+k8sRequestsMemory: "0.1Gi"
+k8sLimitsCpu: "0.8"
+k8sLimitsMemory: "0.8Gi"
+
+logLevel: INFO
+uriBase: "file:///data/"
+
+kafkaConfigs: prod-kafka-bootstrap-servers
+# TODO: To be changed
+inputTopicName: fedora-output-json-records
+# TODO: To be changed
+reportingTopicName: import-process-reporting
+groupId: stage-media-metadata-indexer
+kafkaConnectionRetries: 2
+
+mariadbDatabaseConfigs: stage-mariadb-db-medienserver-configs
+mariadbUserSecrets: stage-mariadb-user-medienserver-secrets
+mariadbConnectionRetries: 2

helm-charts/values-test.yaml

+registry: "cr.gitlab.switch.ch"
+image: "memoriav/memobase-2020/services/postprocessing/mediametadatatodb"
+tag: "latest"
+
+k8sName: media-metadata-indexer
+k8sNamespace: memobase
+k8sGroupId: di
+k8sGroupName: documents-import
+k8sEnvironment: test
+k8sReplicas: 1
+k8sRequestsCpu: "0.1"
+k8sRequestsMemory: "0.1Gi"
+k8sLimitsCpu: "0.8"
+k8sLimitsMemory: "0.8Gi"
+
+logLevel: INFO
+uriBase: "file:///data/"
+
+kafkaConfigs: test-kafka-bootstrap-servers
+# TODO: To be changed
+inputTopicName: fedora-output-json-records
+# TODO: To be changed
+reportingTopicName: import-process-reporting
+groupId: test-media-metadata-indexer
+kafkaConnectionRetries: 2
+
+mariadbDatabaseConfigs: test-mariadb-db-medienserver-configs
+mariadbUserSecrets: test-mariadb-user-medienserver-secrets
+mariadbConnectionRetries: 2

k8s-manifests/test/deployment.yml

-apiVersion: apps/v1
-kind: Deployment
-metadata:
-  name: pp-mediametadatatodb-deployment
-  namespace: memobase
-  labels:
-    app: pp-mediametadatatodb-app
-spec:
-  selector:
-    matchLabels:
-      app: pp-mediametadatatodb-app
-  replicas: 1
-  template:
-    metadata:
-      labels:
-        app: pp-mediametadatatodb-app
-        tier: web
-    spec:
-      containers:
-          - name: pp-mediametadatatodb-container
-            image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/postprocessing/mediametadatatodb:latest
-            env:
-              - name: INPUT_TOPIC
-                value: 'fedora-output-json-records'
-              - name: KAFKA_CONNECTION_RETRIES
-                value: '2'
-              - name: KAFKA_GROUP_ID
-                value: 'medienserverMetadataService'
-              - name: LOG_LEVEL
-                value: 'INFO'
-              - name: MARIADB_CONNECTION_RETRIES
-                value: '2'
-              - name: MARIADB_DATABASE
-                value: 'medienserver'
-              - name: MARIADB_HOST
-                value: 'mb-db1.memobase.unibas.ch'
-              - name: MARIADB_PORT
-                value: '3306'
-              - name: REPORTING_TOPIC
-                value: 'import-process-reporting'
-              - name: URI_BASE
-                value: 'file:///data/'
-            envFrom:
-              - configMapRef:
-                  name: prod-kafka-bootstrap-servers
-              - secretRef:
-                  name: mariadb-mb-db1-medienserver
-            imagePullPolicy: Always

mediametadatatodb_app/resources/MediametadataToDB.py

@@ -36,12 +36,12 @@ def _connect_to_kafka(retries=0):
     """
     try:
         consumer = KafkaConsumer(
-            os.environ['INPUT_TOPIC'],
+            os.environ['TOPIC_IN'],
             value_deserializer=lambda m: json.loads(m.decode('utf8')),
             bootstrap_servers=os.environ['KAFKA_BOOTSTRAP_SERVERS'],
             auto_offset_reset='earliest',
             enable_auto_commit=False,
-            group_id=os.environ['KAFKA_GROUP_ID'],
+            group_id=os.environ['GROUP_ID'],
             consumer_timeout_ms=30000)
         return consumer
     except KafkaError as ex:

mediametadatatodb_app/resources/reporter.py

@@ -55,8 +55,8 @@ class Reporter:
             }
             json_report = json.dumps(report)
             value_bytes = bytes(json_report, encoding='utf-8')
-            logging.debug(f'Sending report to {os.environ["REPORTING_TOPIC"]}: {json_report}')
-            self._producer.send(os.environ['REPORTING_TOPIC'], key=key_bytes, value=value_bytes,
+            logging.debug(f'Sending report to {os.environ["TOPIC_PROCESS"]}: {json_report}')
+            self._producer.send(os.environ['TOPIC_PROCESS'], key=key_bytes, value=value_bytes,
                                 headers=headers)
             self._producer.flush()
         except Exception as ex: