Commit bebae217 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Fixed deployment and added logging for start.

parent c2f8c4da
......@@ -8,6 +8,7 @@ TFV_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/
TDT_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/table-data-transform
MAPPER_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/mapper-service
MEDIA_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/media-linker
MME_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/media-metadata-extractor
NORM_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/normalization-service
FEDORA_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/fedora-ingest-service
......@@ -25,6 +26,9 @@ helm chart export ${MAPPER_REGISTRY}:${MAPPER_CHART_VERSION} -d charts/
helm chart pull ${MEDIA_REGISTRY}:${MEDIA_CHART_VERSION}
helm chart export ${MEDIA_REGISTRY}:${MEDIA_CHART_VERSION} -d charts/
helm chart pull ${MME_REGISTRY}:${MME_CHART_VERSION}
helm chart export ${MME_REGISTRY}:${MME_CHART_VERSION} -d charts/
helm chart pull ${NORM_REGISTRY}:${NORM_CHART_VERSION}
helm chart export ${NORM_REGISTRY}:${NORM_CHART_VERSION} -d charts/
......
......@@ -7,6 +7,7 @@ python topic-creation.py delete
helm uninstall ${NAME}-text-file-validation
helm uninstall ${NAME}-table-data-transform
helm uninstall ${NAME}-mapper-service
helm uninstall ${NAME}-media-metadata-extractor
helm uninstall ${NAME}-normalization-service
helm uninstall ${NAME}-fedora-ingest-service
kubectl delete configmap ${PROCESS_ID}-mapper-service-mapping-config
......
......@@ -26,13 +26,23 @@ helm install ${NAME}-mapper-service ./charts/mapper-service\
--set recordSetId=${RECORD_SET_ID}\
--set configMapName=${PROCESS_ID}-mapper-service-mapping-config
helm install ${NAME}-media-linker ./charts/media-linker\
--set processId=${PROCESS_ID}\
--set institutionId=${INSTITUTION_ID}\
--set recordSetId=${RECORD_SET_ID}
helm install ${NAME}-media-metadata-extractor ./charts/media-metadata-extractor\
--set processId=${PROCESS_ID}\
--set institutionId=${INSTITUTION_ID}\
--set recordSetId=${RECORD_SET_ID}
kubectl create configmap ${PROCESS_ID}-normalization-service-local-transform-config --from-file ./configs/localTransforms.yml
helm install ${NAME}-normalization-service ./charts/normalization-service\
--set processId=${PROCESS_ID}\
--set institutionId=${INSTITUTION_ID}\
--set recordSetId=${RECORD_SET_ID}\
--set lastJobName=mapper-service\
--set lastJobName=media-metadata-extractor\
--set hasLocalTransformConfig=true
helm install ${NAME}-fedora-ingest-service ./charts/fedora-ingest-service\
......
......@@ -2,8 +2,8 @@
TFV_CHART_VERSION="0.4.0-chart"
TDT_CHART_VERSION="0.4.5-chart"
MAPPER_CHART_VERSION="0.12.0-chart"
MAPPER_CHART_VERSION="0.12.1-chart"
MEDIA_CHART_VERSION="0.1.3-chart"
MME_CHART_VERSION="0.1.1-chart"
MME_CHART_VERSION="0.1.7-chart"
NORM_CHART_VERSION="0.4.1-chart"
INGESTER_CHART_VERSION="0.2.1-chart"
......@@ -15,7 +15,7 @@ class HelmChart:
wait {bool} -- Whether to wait for all pods to be ready (default: {True})
"""
def __init__(self, name: str, chart: str, namespace: str = None,
fail_on_error: bool = True, wait: bool = True):
fail_on_error: bool = False, wait: bool = True):
self.name = name
self.chart = chart
self.namespace = namespace
......@@ -65,7 +65,7 @@ class HelmChart:
return self._exec_command(helm_cmd, self.fail_on_error)
@staticmethod
def _exec_command(cmd, fail_on_err=True) -> CompletedProcess:
def _exec_command(cmd: list, fail_on_error: bool = False) -> CompletedProcess:
base_cmd = [
"helm",
]
......@@ -74,8 +74,7 @@ class HelmChart:
base_cmd + cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=fail_on_err,
text=True
check=fail_on_error
)
......
from kubernetes.client import CoreV1Api
from string import Template
import subprocess
from subprocess import CompletedProcess
class ConfigMap:
......@@ -7,20 +7,24 @@ class ConfigMap:
def __init__(self, **kwargs):
self.name = kwargs['name']
self.path = kwargs['path']
self.k8s = CoreV1Api()
with open('api/data/config_map_template.yml', 'r') as fp:
self.template = Template(fp.read())
def create(self, process_id: str):
with open(self.path, 'r') as fp:
content = fp.read()
body = Template(content).substitute(name=f'{process_id}-{self.name}', namespace='memobase')
# body = self.template.substitute(name=f'{process_id}-{self.name}',
# namespace='memobase',
# file_name=self.path.split('/')[-1],
# content=content)
print(body)
self.k8s.create_namespaced_config_map(namespace='memobase', body=body)
def create(self, process_id: str) -> CompletedProcess:
cmd = ["create", "configmap", f'{process_id}-{self.name}', "--from-file", self.path]
return self._exec_command(cmd, False)
def delete(self, process_id: str):
self.k8s.delete_namespaced_config_map(name=f'{process_id}-{self.name}', namespace='memobase')
def delete(self, process_id: str) -> CompletedProcess:
cmd = ["delete", "configmap", f'{process_id}-{self.name}']
return self._exec_command(cmd, False)
@staticmethod
def _exec_command(cmd, fail_on_err=True) -> CompletedProcess:
base_cmd = [
"kubectl",
]
# for debug print (' '.join(base_cmd+cmd))
return subprocess.run(
base_cmd + cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=fail_on_err
)
......@@ -2,6 +2,7 @@ from api.kafka import KafkaTopics
from api.helm import all_charts
from api.kubernetes import ConfigMap
import json
import logging
class ImportProcess:
......@@ -17,12 +18,23 @@ class ImportProcess:
def start(self):
self.topics.create()
for config_map in self.config_maps:
config_map.create(self.process_id)
completed_process = config_map.create(self.process_id)
if completed_process.returncode:
logging.error(completed_process.stderr.decode('utf-8'))
exit(1)
else:
logging.info(f"Deployed config map {self.process_id}-{config_map.name}.")
values = self.values['global']
for chart in self.charts:
if chart.name in self.values:
values += self.values[chart.name]
chart.install(self.process_id, **values)
for key in self.values[chart.name]:
values[key] = self.values[chart.name][key]
completed_process = chart.install(self.process_id, values)
if completed_process.returncode:
logging.error(completed_process.stderr.decode('utf-8'))
exit(1)
else:
logging.info(f"Installed chart {chart.name}.")
def remove(self):
for chart in self.charts:
......
apiVersion: v1
kind: ConfigMap
metadata:
name: ${name}
namespace: ${namespace}
data:
localTransforms.yml: |
splitEntity:
- type: skos:Concept
property: skos:prefLabel
delimiter: ","
- type: rico:Agent
property: rico:name
delimiter: ","
- type: rico:Place
property: rico:name
delimiter: ","
- type: rico:Language
property: rico:name
delimiter: ","
\ No newline at end of file
splitEntity:
- type: skos:Concept
property: skos:prefLabel
delimiter: ","
- type: rico:Agent
property: rico:name
delimiter: ","
- type: rico:Place
property: rico:name
delimiter: ","
- type: rico:Language
property: rico:name
delimiter: ","
\ No newline at end of file
import json
from kubernetes.config import load_kube_config
from api.process import ImportProcess
import logging
import sys
import os
print(os.getcwd())
load_kube_config()
logging.basicConfig(stream=sys.stdout, level=logging.INFO)
if __name__ == '__main__':
with open('./configurations/becker/setup.json', 'r') as fp:
......
apiVersion: v1
kind: ConfigMap
metadata:
name: p0001-normalization-service-local-transform-config
namespace: memobase
data:
localTransforms.yml: |
splitEntity:
- type: skos:Concept
property: skos:prefLabel
delimiter: ","
- type: rico:Agent
property: rico:name
delimiter: ","
- type: rico:Place
property: rico:name
delimiter: ","
- type: rico:Language
property: rico:name
delimiter: ","
kafka-python
rdflib
kubernetes
\ No newline at end of file
rdflib
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment