Skip to content

GitLab

Commit 25fdc73b authored by Günter Hipler's avatar Günter Hipler
Browse files

Merge branch 'helm-charts' into 'master' 

Replace k8s manifest with helm chart

See merge request !1
parents ffad8b46 0c1b5e5d
Pipeline #31288 passed with stages
in 7 minutes and 49 seconds
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 2929 additions and 80 deletions
.gitlab-ci.yml
View file @ 25fdc73b
......@@ -11,4 +11,5 @@ include:
file: 'sbt-build/sbt-build-2.13.yml'
- project: 'memoriav/memobase-2020/utilities/ci-templates'
file: 'docker-image/docker-image.yml'
\ No newline at end of file
- project: 'memoriav/memobase-2020/utilities/ci-templates'
file: 'helm-chart/helm-chart.yml'
bin/configmap.es.kafka.minikube.yaml 0 → 100644
View file @ 25fdc73b
apiVersion: v1
kind: ConfigMap
metadata:
name: prod-kafka-bootstrap-servers
namespace: memobase
data:
# property-like keys; each key maps to a simple value
KAFKA_BOOTSTRAP_SERVERS: "host.minikube.internal:9092,host.minikube.internal:9093,host.minikube.internal:9094"
---
apiVersion: v1
kind: ConfigMap
metadata:
namespace: memobase
name: prod-institutions-index
data:
ELASTIC_INDEX: institutions-v6
---
apiVersion: v1
kind: ConfigMap
metadata:
name: prod-record-sets-index
namespace: memobase
data:
ELASTIC_INDEX: record-sets-v7
---
apiVersion: v1
data:
ELASTIC_HOST: 192.168.1.53
ELASTIC_PORT: "8080"
kind: ConfigMap
metadata:
name: prod-elastic-configs
namespace: memobase
bin/create.configmaps.sh 0 → 100755
View file @ 25fdc73b
#!/usr/bin/env bash
namespace=memobase
cd ../configs
kubectl create cm di-instcoordmapping-prod-config --namespace=$namespace --from-file ./institutions_coordinates_mapping/labels.csv
kubectl create cm di-instcoordmapping-test-config --namespace=$namespace --from-file ./institutions_coordinates_mapping/labels.csv
kubectl create cm di-instcoordmapping-stage-config --namespace=$namespace --from-file ./institutions_coordinates_mapping/labels.csv
kubectl create cm di-isocode693mapping-prod-config --namespace=$namespace --from-file ./isocode-693-mapping/labels.csv
kubectl create cm di-isocode693mapping-test-config --namespace=$namespace --from-file ./isocode-693-mapping/labels.csv
kubectl create cm di-isocode693mapping-stage-config --namespace=$namespace --from-file ./isocode-693-mapping/labels.csv
kubectl create cm di-oaiexportdefinitions-prod-config --namespace=$namespace --from-file ./oai_export_definitions/rules.json
kubectl create cm di-oaiexportdefinitions-test-config --namespace=$namespace --from-file ./oai_export_definitions/rules.json
kubectl create cm di-oaiexportdefinitions-stage-config --namespace=$namespace --from-file ./oai_export_definitions/rules.json
bin/delete.configmaps.sh 0 → 100755
View file @ 25fdc73b
#!/usr/bin/env bash
namespace=memobase
kubectl delete cm di-instcoordmapping-prod-config --namespace=$namespace
kubectl delete cm di-instcoordmapping-test-config --namespace=$namespace
kubectl delete cm di-instcoordmapping-stage-config --namespace=$namespace
kubectl delete cm di-isocode693mapping-prod-config --namespace=$namespace
kubectl delete cm di-isocode693mapping-test-config --namespace=$namespace
kubectl delete cm di-isocode693mapping-stage-config --namespace=$namespace
kubectl delete cm di-oaiexportdefinitions-prod-config --namespace=$namespace
kubectl delete cm di-oaiexportdefinitions-test-config --namespace=$namespace
kubectl delete cm di-oaiexportdefinitions-stage-config --namespace=$namespace
gh/es.mappings.documents.json 0 → 100644
View file @ 25fdc73b
This diff is collapsed.
gh/es.mappings.institutions.json 0 → 100644
View file @ 25fdc73b
{
"settings": {
"number_of_replicas": 1,
"number_of_shards": 3
},
"mappings": {
"dynamic_templates": [
{
"filter_as_keyword": {
"path_match": "*.filter",
"mapping": {"type": "keyword"}
}
},
{
"facet_as_keyword": {
"path_match": "*.facet",
"mapping": {"type": "keyword"}
}
},
{
"add_sort_field_to_name": {
"path_match": "name.*",
"mapping": {
"fields": {
"sort": {"type": "keyword"}
}
}
}
},
{
"german_fields": {
"path_match": "*.de",
"mapping": {
"analyzer": "german",
"fields": {
"filter": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
{
"french_fields": {
"path_match": "*.fr",
"mapping": {
"analyzer": "french",
"fields": {
"filter": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
{
"italian_fields": {
"path_match": "*.it",
"mapping": {
"analyzer": "italian",
"fields": {
"filter": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
},
{
"unknown_language_fields": {
"path_match": "*.un",
"mapping": {
"fields": {
"filter": {
"ignore_above": 256,
"type": "keyword"
}
},
"type": "text"
}
}
}
],
"date_detection": false,
"properties": {
"address": {"type": "text"},
"canton": {
"properties": {
"name": {
"properties": {
"de": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "german"
},
"fr": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "french"
},
"it": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "italian"
}
}
}
}
},
"city": {
"properties": {
"de": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "german"
},
"fr": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "french"
},
"it": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "italian"
},
"un": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"description": {
"properties": {
"de": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "german"
},
"fr": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "french"
},
"it": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "italian"
},
"un": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
}
}
}
},
"documentType": {
"properties": {
"filter": {"type": "keyword"},
"name": {
"properties": {
"de": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "german"
},
"fr": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "french"
},
"it": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "italian"
}
}
}
}
},
"id": {"type": "keyword"},
"isPublished": {"type": "boolean"},
"keyVisualLink": {"type": "keyword"},
"lastUpdatedDate": {
"type": "keyword",
"fields": {
"sort": {"type": "date"}
}
},
"name": {
"properties": {
"de": {
"type": "text",
"fields": {
"sort": {"type": "keyword"}
}
},
"fr": {
"type": "text",
"fields": {
"sort": {"type": "keyword"}
}
},
"it": {
"type": "text",
"fields": {
"sort": {"type": "keyword"}
}
}
}
},
"numberOfDocuments": {"type": "long"},
"numberOfDocumentsPublished": {"type": "long"},
"numberOfRecordSets": {"type": "long"},
"postalCodes": {"type": "text"},
"published": {"type": "boolean"},
"teaserColor": {"type": "keyword"},
"teaserColorComputed": {"type": "keyword"},
"type": {
"properties": {
"filter": {"type": "keyword"},
"name": {
"properties": {
"de": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "german"
},
"fr": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "french"
},
"it": {
"type": "text",
"fields": {
"filter": {
"type": "keyword",
"ignore_above": 256
}
},
"analyzer": "italian"
}
}
}
}
}
}
}
}
gh/es.mappings.json
View file @ 25fdc73b
{
"index_patterns": [
"oai*"
],
"settings": {
"number_of_replicas": 1,
"number_of_shards": 3
......@@ -9,30 +6,21 @@
"mappings": {
"date_detection": false,
"properties": {
"id": {
"type": "keyword"
},
"document": {
"type": "keyword",
"index": false
},
"format": {
"type": "keyword"
},
"published": {
"type": "boolean"
},
"recordset": {
"type": "keyword"
"type": "binary",
"store": true
},
"format": {"type": "keyword"},
"id": {"type": "keyword"},
"institution": {"type": "keyword"},
"lastUpdatedDate": {
"type": "keyword",
"fields": {
"sort": {
"type": "date"
}
"sort": {"type": "date"}
}
}
},
"published": {"type": "boolean"},
"recordset": {"type": "keyword"}
}
}
}
gh/es.mappings.recordset.json 0 → 100644
View file @ 25fdc73b
This diff is collapsed.
gh/start.process.local.txt
View file @ 25fdc73b
......@@ -2,6 +2,6 @@ docker run -d --network host --rm --env-file ./env.txt rico2edm:latest
cd /home/swissbib/environment/code/repositories/memoriav/gitlab/services/elastic-services/elastic-bulk-action-service/gh
docker run -d --network host --rm --env-file ./env.txt elastic-bulk:latest
docker run -d -v /home/swissbib/environment/code/repositories/memoriav/gitlab/services/postprocessing/rico-edm-transformer/configs:/configs --network host --rm --env-file ./env.txt rico2edm:latest
\ No newline at end of file
helm-charts/.helmignore 0 → 100644
View file @ 25fdc73b
# Patterns to ignore when building packages.
# This supports shell glob matching, relative path matching, and
# negation (prefixed with !). Only one pattern per line.
.DS_Store
# Common VCS dirs
.git/
.gitignore
.bzr/
.bzrignore
.hg/
.hgignore
.svn/
# Common backup files
*.swp
*.bak
*.tmp
*.orig
*~
# Various IDEs
.project
.idea/
*.tmproj
.vscode/
\ No newline at end of file
helm-charts/Chart.yaml 0 → 100644
View file @ 25fdc73b
apiVersion: v2
name: rico-edm-transformer
description: A helm chart for the RiCO EDM Transformer
type: application
version: 0.0.0
appVersion: 0.0.0
maintainers:
- name: Sebastian Schüpbach
email: sebastian.schuepbach@unibas.ch
helm-charts/helm-values/di-rico-edm-transformer-prod.yaml 0 → 100644
View file @ 25fdc73b
tag: 0.3.1
k8sEnvironment: prod
k8sReplicas: 1
k8sRequestsCpu: "0.2"
k8sRequestsMemory: "128Mi"
k8sLimitsCpu: "0.4"
k8sLimitsMemory: "256Mi"
kafkaConfigs: prod-kafka-bootstrap-servers
inputTopicName: mb-di-processed-records-prod
outputTopicName: mb-di-edm-es-records-prod
reportingTopicName: mb-di-reporting-prod
applicationId: prod-rico-edm-transformer
elasticHostConfigs: prod-elastic-configs
institutionIndexCM: prod-institutions-index
recordSetIndexCM: prod-record-sets-index
helm-charts/helm-values/di-rico-edm-transformer-stage.yaml 0 → 100644
View file @ 25fdc73b
tag: "0.3.1"
k8sEnvironment: stage
k8sReplicas: 1
k8sRequestsCpu: "0.2"
k8sRequestsMemory: "128Mi"
k8sLimitsCpu: "0.4"
k8sLimitsMemory: "256Mi"
kafkaConfigs: prod-kafka-bootstrap-servers
inputTopicName: mb-di-processed-records-stage
outputTopicName: mb-di-edm-es-records-stage
reportingTopicName: mb-di-reporting-stage
applicationId: stage-rico-edm-transformer
elasticHostConfigs: stage-elastic-configs
institutionIndexCM: stage-institutions-index
recordSetIndexCM: stage-record-sets-index
helm-charts/helm-values/di-rico-edm-transformer-test.yaml 0 → 100644
View file @ 25fdc73b
tag: "0.3.1"
k8sEnvironment: test
k8sReplicas: 1
k8sRequestsCpu: "0.2"
k8sRequestsMemory: "128Mi"
k8sLimitsCpu: "0.4"
k8sLimitsMemory: "256Mi"
kafkaConfigs: test-kafka-bootstrap-servers
inputTopicName: mb-di-processed-records-prod
outputTopicName: mb-di-edm-es-records-prod
reportingTopicName: mb-di-reporting-prod
applicationId: test-rico-edm-transformer
elasticHostConfigs: test-elastic-configs
institutionIndexCM: test-institutions-index
recordSetIndexCM: test-record-sets-index
helm-charts/templates/app-config.yaml 0 → 100644
View file @ 25fdc73b
apiVersion: v1
kind: ConfigMap
metadata:
name: "{{ .Values.k8sGroupId }}-{{ .Values.k8sName }}-{{ .Values.k8sEnvironment}}-config"
namespace: "{{ .Values.k8sNamespace }}"
data:
APPLICATION_ID: "{{ .Values.applicationId}}"
TOPIC_IN: "{{ .Values.inputTopicName }}"
TOPIC_OUT: "{{ .Values.outputTopicName }}"
TOPIC_PROCESS: "{{ .Values.reportingTopicName }}"
ISOCODE_MAPPING: "{{ .Values.isocodeMapping }}"
INSTITUTIONS_COORDINATES: "{{ .Values.institutionsCoordinates }}"
EXPORT_DEFINITION_RULES: "{{ .Values.exportDefinitionRules }}"