Unverified Commit 3bb4c31c authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files
parent 0e2bca5f
Pipeline #24533 passed with stages
in 2 minutes and 10 seconds
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
\ No newline at end of file
apiVersion: v2
name: iiif-manifest-indexer
description: A helm chart for the Media Metadata Indexer
type: application
version: 0.0.0
appVersion: 0.0.0
maintainers:
- name: Sebastian Schüpbach
email: sebastian.schuepbach@unibas.ch
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
namespace: "{{ .Values.k8sNamespace }}"
data:
LOG_LEVEL: "{{ .Values.logLevel }}"
URI_BASE: "{{ .Values.uriBase }}"
GROUP_ID: "{{ .Values.groupId}}"
TOPIC_IN: "{{ .Values.inputTopicName }}"
TOPIC_PROCESS: "{{ .Values.reportingTopicName }}"
KAFKA_CONNECTION_RETRIES: "{{ .Values.kafkaConnectionRetries }}"
MARIADB_CONNECTION_RETRIES: "{{ .Values.mariadbConnectionRetries }}"
\ No newline at end of file
apiVersion: apps/v1
kind: Deployment
metadata:
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}"
namespace: "{{ .Values.k8sNamespace }}"
labels:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
spec:
selector:
matchLabels:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
replicas: { { .Values.k8sReplicas } }
template:
metadata:
labels:
app: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-app"
group: "{{ .Values.k8sGroupName }}"
environment: "{{ .Values.k8sEnvironment }}"
spec:
containers:
- name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-container"
image: "{{ .Values.registry }}/{{ .Values.image }}:{{ .Values.tag }}"
imagePullPolicy: Always
resources:
requests:
cpu: "{{ .Values.k8sRequestsCpu }}"
memory: "{{ .Values.k8sRequestsMemory }}"
limits:
cpu: "{{ .Values.k8sLimitsCpu }}"
memory: "{{ .Values.k8sLimitsMemory }}"
envFrom:
- configMapRef:
name: "{{ .Values.kafkaConfigs }}"
- configMapRef:
name: "{{ .Values.mariadbDatabaseConfigs }}"
- configMapRef:
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
env:
- name: MARIADB_PASSWORD
valueFrom:
secretKeyRef:
name: { { .Values.mariadbUserSecrets } }
key: MARIADB_PASSWORD
- name: MARIADB_USER
valueFrom:
secretKeyRef:
name: { { .Values.mariadbUserSecrets } }
key: MARIADB_USER
restartPolicy: Always
registry: "cr.gitlab.switch.ch"
image: "memoriav/memobase-2020/services/postprocessing/mediametadatatodb"
tag: "latest"
k8sName: media-metadata-indexer
k8sNamespace: memobase
k8sGroupId: di
k8sGroupName: documents-import
k8sEnvironment: prod
k8sReplicas: 1
k8sRequestsCpu: "0.1"
k8sRequestsMemory: "0.1Gi"
k8sLimitsCpu: "0.8"
k8sLimitsMemory: "0.8Gi"
logLevel: INFO
uriBase: "file:///data/"
kafkaConfigs: prod-kafka-bootstrap-servers
# TODO: To be changed
inputTopicName: fedora-output-json-records
# TODO: To be changed
reportingTopicName: import-process-reporting
groupId: prod-media-metadata-indexer
kafkaConnectionRetries: 2
mariadbDatabaseConfigs: prod-mariadb-db-medienserver-configs
mariadbUserSecrets: prod-mariadb-user-medienserver-secrets
mariadbConnectionRetries: 2
registry: "cr.gitlab.switch.ch"
image: "memoriav/memobase-2020/services/postprocessing/mediametadatatodb"
tag: "latest"
k8sName: media-metadata-indexer
k8sNamespace: memobase
k8sGroupId: di
k8sGroupName: documents-import
k8sEnvironment: stage
k8sReplicas: 1
k8sRequestsCpu: "0.1"
k8sRequestsMemory: "0.1Gi"
k8sLimitsCpu: "0.8"
k8sLimitsMemory: "0.8Gi"
logLevel: INFO
uriBase: "file:///data/"
kafkaConfigs: prod-kafka-bootstrap-servers
# TODO: To be changed
inputTopicName: fedora-output-json-records
# TODO: To be changed
reportingTopicName: import-process-reporting
groupId: stage-media-metadata-indexer
kafkaConnectionRetries: 2
mariadbDatabaseConfigs: stage-mariadb-db-medienserver-configs
mariadbUserSecrets: stage-mariadb-user-medienserver-secrets
mariadbConnectionRetries: 2
registry: "cr.gitlab.switch.ch"
image: "memoriav/memobase-2020/services/postprocessing/mediametadatatodb"
tag: "latest"
k8sName: media-metadata-indexer
k8sNamespace: memobase
k8sGroupId: di
k8sGroupName: documents-import
k8sEnvironment: test
k8sReplicas: 1
k8sRequestsCpu: "0.1"
k8sRequestsMemory: "0.1Gi"
k8sLimitsCpu: "0.8"
k8sLimitsMemory: "0.8Gi"
logLevel: INFO
uriBase: "file:///data/"
kafkaConfigs: test-kafka-bootstrap-servers
# TODO: To be changed
inputTopicName: fedora-output-json-records
# TODO: To be changed
reportingTopicName: import-process-reporting
groupId: test-media-metadata-indexer
kafkaConnectionRetries: 2
mariadbDatabaseConfigs: test-mariadb-db-medienserver-configs
mariadbUserSecrets: test-mariadb-user-medienserver-secrets
mariadbConnectionRetries: 2
apiVersion: apps/v1
kind: Deployment
metadata:
name: pp-mediametadatatodb-deployment
namespace: memobase
labels:
app: pp-mediametadatatodb-app
spec:
selector:
matchLabels:
app: pp-mediametadatatodb-app
replicas: 1
template:
metadata:
labels:
app: pp-mediametadatatodb-app
tier: web
spec:
containers:
- name: pp-mediametadatatodb-container
image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/postprocessing/mediametadatatodb:latest
env:
- name: INPUT_TOPIC
value: 'fedora-output-json-records'
- name: KAFKA_CONNECTION_RETRIES
value: '2'
- name: KAFKA_GROUP_ID
value: 'medienserverMetadataService'
- name: LOG_LEVEL
value: 'INFO'
- name: MARIADB_CONNECTION_RETRIES
value: '2'
- name: MARIADB_DATABASE
value: 'medienserver'
- name: MARIADB_HOST
value: 'mb-db1.memobase.unibas.ch'
- name: MARIADB_PORT
value: '3306'
- name: REPORTING_TOPIC
value: 'import-process-reporting'
- name: URI_BASE
value: 'file:///data/'
envFrom:
- configMapRef:
name: prod-kafka-bootstrap-servers
- secretRef:
name: mariadb-mb-db1-medienserver
imagePullPolicy: Always
......@@ -36,12 +36,12 @@ def _connect_to_kafka(retries=0):
"""
try:
consumer = KafkaConsumer(
os.environ['INPUT_TOPIC'],
os.environ['TOPIC_IN'],
value_deserializer=lambda m: json.loads(m.decode('utf8')),
bootstrap_servers=os.environ['KAFKA_BOOTSTRAP_SERVERS'],
auto_offset_reset='earliest',
enable_auto_commit=False,
group_id=os.environ['KAFKA_GROUP_ID'],
group_id=os.environ['GROUP_ID'],
consumer_timeout_ms=30000)
return consumer
except KafkaError as ex:
......
......@@ -55,8 +55,8 @@ class Reporter:
}
json_report = json.dumps(report)
value_bytes = bytes(json_report, encoding='utf-8')
logging.debug(f'Sending report to {os.environ["REPORTING_TOPIC"]}: {json_report}')
self._producer.send(os.environ['REPORTING_TOPIC'], key=key_bytes, value=value_bytes,
logging.debug(f'Sending report to {os.environ["TOPIC_PROCESS"]}: {json_report}')
self._producer.send(os.environ['TOPIC_PROCESS'], key=key_bytes, value=value_bytes,
headers=headers)
self._producer.flush()
except Exception as ex:
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment