Skip to content

GitLab

Verified Commit 14582288 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

build distinct chart for each newly built job

parent 1a96e564
Pipeline #37740 passed with stages
in 1 minute and 7 seconds
Hide whitespace changes
Inline Side-by-side
helm-charts/configmap/prod/gunicorn.conf.py
View file @ 14582288
......@@ -4,24 +4,24 @@ accesslog = "/dev/stdout"
access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
disable_redirect_access_to_syslog = True
errorlog = "/dev/stderr"
loglevel = 'info'
loglevel = "info"
workers = 1
logconfig_dict = dict(
version=1,
disable_existing_loggers=True,
loggers={
"gunicorn.error": {
"level": "INFO",
"handlers": ["error_console"],
"propagate": False,
"qualname": "gunicorn.error"
}
},
formatters={
"generic": {
"format": "[%(levelname)s] [%(name)s] %(message)s",
"datefmt": "[%Y-%m-%d %H:%M:%S %z]",
"class": "logging.Formatter"
}
version=1,
disable_existing_loggers=True,
loggers={
"gunicorn.error": {
"level": "INFO",
"handlers": ["error_console"],
"propagate": False,
"qualname": "gunicorn.error",
}
},
formatters={
"generic": {
"format": "[%(levelname)s] [%(name)s] %(message)s",
"datefmt": "[%Y-%m-%d %H:%M:%S %z]",
"class": "logging.Formatter",
}
},
)
helm-charts/configmap/test/gunicorn.conf.py
View file @ 14582288
......@@ -4,24 +4,24 @@ accesslog = "/dev/stdout"
access_log_format = '%(h)s %(l)s %(u)s %(t)s "%(r)s" %(s)s %(b)s "%(f)s" "%(a)s"'
disable_redirect_access_to_syslog = True
errorlog = "/dev/stderr"
loglevel = 'debug'
loglevel = "debug"
workers = 1
logconfig_dict = dict(
version=1,
disable_existing_loggers=True,
loggers={
"gunicorn.error": {
"level": "DEBUG",
"handlers": ["error_console"],
"propagate": False,
"qualname": "gunicorn.error"
}
},
formatters={
"generic": {
"format": "[%(levelname)s] [%(name)s] %(message)s",
"datefmt": "[%Y-%m-%d %H:%M:%S %z]",
"class": "logging.Formatter"
}
version=1,
disable_existing_loggers=True,
loggers={
"gunicorn.error": {
"level": "DEBUG",
"handlers": ["error_console"],
"propagate": False,
"qualname": "gunicorn.error",
}
},
formatters={
"generic": {
"format": "[%(levelname)s] [%(name)s] %(message)s",
"datefmt": "[%Y-%m-%d %H:%M:%S %z]",
"class": "logging.Formatter",
}
},
)
import_api_app/helpers/delete_service.py
View file @ 14582288
......@@ -33,7 +33,7 @@ class DeleteService:
"deleteId": record_id,
}
try:
return self.run_job(set_values, dryrun)
return self.run_job(set_values, dryrun, "record")
except Exception as ex:
message = str(ex)
self.logger.error(message)
......@@ -46,7 +46,7 @@ class DeleteService:
"deleteId": recordset_id,
}
try:
return self.run_job(set_values, dryrun)
return self.run_job(set_values, dryrun, "recordset")
except Exception as ex:
message = str(ex)
self.logger.error(message)
......@@ -59,30 +59,24 @@ class DeleteService:
"deleteId": institution_id,
}
try:
return self.run_job(set_values, dryrun)
return self.run_job(set_values, dryrun, "institution")
except Exception as ex:
message = str(ex)
self.logger.error(message)
raise ImportApiError(message)
def run_job(self, set_values: dict, dryrun: bool) -> str:
res_install = self.do_helm_install(set_values, dryrun)
if res_install.returncode > 0:
self.logger.error(
"Something went when installing helm chart: returncode: {}; stdout: {}; stderr: {}".format(
res_install.returncode, res_install.stdout, res_install.stderr
)
)
self.logger.debug(
"Cleaning up helm chart " + set_values["sessionId"] + "-deleter"
chart_name = "dd-marker-{}-{}".format(
os.getenv("ENV", ""), set_values["sessionId"]
)
res_uninstall = self.helm.uninstall(
set_values["sessionId"] + "-deleter", self.namespace
)
if res_uninstall.returncode > 0:
res_install = self.do_helm_install(set_values, chart_name, dryrun)
if res_install.returncode > 0:
self.logger.error(
"Something went when uninstalling helm chart: returncode: {}; stdout: {}; stderr: {}".format(
res_uninstall.returncode, res_uninstall.stdout, res_uninstall.stderr
"Something went when installing helm chart {}: returncode: {}; stdout: {}; stderr: {}".format(
chart_name,
res_install.returncode,
res_install.stdout,
res_install.stderr,
)
)
retJson = {
......@@ -92,7 +86,7 @@ class DeleteService:
}
return json.dumps(retJson, ensure_ascii=True, indent=2)
def do_helm_install(self, set_values, dryrun):
def do_helm_install(self, set_values, chart_name, dryrun):
self.logger.error(
"calling delete service: type="
+ set_values["deleteObject"]
......@@ -101,9 +95,8 @@ class DeleteService:
+ " / session="
+ set_values["sessionId"]
)
# We need to handle the fact that we can't use an array as argument
# value when calling helm. For details see
# https://github.com/helm/helm/issues/1987#issuecomment-682352463.
# Due to limitations on arrays in helm argument values, we need to build a string
# (which will afterwards be converted back to an array by the k8s template)
job_args = [
"--" + set_values["deleteObject"] + "-filter",
set_values["deleteId"],
......@@ -112,7 +105,11 @@ class DeleteService:
job_args.extend(["--dry-run", set_values["sessionId"]])
else:
job_args.append(set_values["sessionId"])
job_args = {"jobArgs": "'" + "' '".join(job_args) + "'"}
# The sessionId is additionally used to build a distinct job id
job_args = {
"jobArgs": "'" + "' '".join(job_args) + "'",
"sessionId": set_values["sessionId"],
}
path_to_charts = os.path.join(self.root_path, "charts", "deletion-marker")
return self.helm.install( # noqa: F841
chart=path_to_charts,
......@@ -121,7 +118,7 @@ class DeleteService:
"helm-values",
"dd-marker-" + os.getenv("ENV", "") + ".yaml",
),
name=set_values["sessionId"] + "-deleter",
name=chart_name,
namespace=self.namespace,
set_values=job_args,
fail_on_err=False,
......
import_api_app/pull-charts.sh
View file @ 14582288
......@@ -8,7 +8,7 @@ TFV_REGISTRY="cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process
TFV_CHART_VERSION="1.3.4-chart"
IPD_REGISTRY="cr.gitlab.switch.ch/memoriav/memobase-2020/services/deletion-components/import-process-delete"
IPD_CHART_VERSION="0.3.5-chart"
IPD_CHART_VERSION="0.3.6-chart"
helm chart pull ${TFV_REGISTRY}:${TFV_CHART_VERSION}
helm chart export ${TFV_REGISTRY}:${TFV_CHART_VERSION} -d charts/
......
import_api_app/wsgi.py
View file @ 14582288
......@@ -37,7 +37,7 @@ environ["HELM_EXPERIMENTAL_OCI"] = "1"
# If app is started via gunicorn
if __name__ != "__main__":
gunicorn_logger = logging.getLogger("gunicorn.error")
gunicorn_logger = logging.getLogger("gunicorn")
app.logger = gunicorn_logger
app.logger.info("Starting production server")
......
setup.py
View file @ 14582288
......@@ -16,15 +16,15 @@
from setuptools import setup
setup(
name='import_api_app',
packages=['import_api_app'],
name="import_api_app",
packages=["import_api_app"],
include_package_data=True,
install_requires=[
'gunicorn',
'flask',
'paramiko',
'requests',
'kafka-python',
'flasgger'
"gunicorn",
"flask",
"paramiko",
"requests",
"kafka-python",
"flasgger",
],
)
tests/test_basic_auth_with_drupal.py
View file @ 14582288
......@@ -21,16 +21,15 @@ from requests.auth import HTTPBasicAuth
class TestBasicAuth(unittest.TestCase):
def test_get_institution(self):
institution_id = '901b3831-3700-4974-a358-ef3e39dc6438'
institution_id = "901b3831-3700-4974-a358-ef3e39dc6438"
url = f"{os.environ['DRUPAL_API_URL']}/de/jsonapi/node/institution/{institution_id}"
headers = {
'X-API-Key': os.environ['DRUPAL_API_KEY']
}
user = os.environ['DRUPAL_USERNAME']
password = os.environ['DRUPAL_PASSWORD']
headers = {"X-API-Key": os.environ["DRUPAL_API_KEY"]}
user = os.environ["DRUPAL_USERNAME"]
password = os.environ["DRUPAL_PASSWORD"]
response = requests.get(url, headers=headers, auth=HTTPBasicAuth(user, password))
response = requests.get(
url, headers=headers, auth=HTTPBasicAuth(user, password)
)
assert response.ok
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment