Commit 07675966 authored by Matthias's avatar Matthias
Browse files

Merge branch 'seb-210824' into 'master'

Improvements

See merge request !2
parents 0d7d5099 98a30f81
Pipeline #33322 passed with stages
in 2 minutes and 19 seconds
......@@ -4,18 +4,18 @@ ARG K8S_VERSION=v1.17.11
ARG HELM_VERSION=v3.6.3
ENV FLASK_APP autodeploy_service_app
ENV FLASK_ENV development
EXPOSE 5000
ENTRYPOINT ["python"]
CMD ["/autodeploy_service_app/main.py"]
ENTRYPOINT ["gunicorn"]
CMD ["--bind", "0.0.0.0:5000", "wsgi:app", "--log-level", "info"]
RUN cd /usr/bin \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl \
&& chmod +x ./kubectl \
&& wget -q https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz \
&& tar -xvzf helm-${HELM_VERSION}-linux-amd64.tar.gz \
&& mv linux-amd64 helm
&& mv linux-amd64/helm helm \
&& rm -rf linux-amd64
WORKDIR /
ADD setup.py /
......
# autodeploy-service
# Autodeploy Service
Service to automatically deploy github projects on k8s.
Deploys Gitlab projects on Kubernetes
Check out the documenation on https://ub-basel.atlassian.net/wiki/spaces/MEMOBASE/pages/352387075/Automatisches+Deployment+auf+Kubernetes
\ No newline at end of file
## Running the server
### Production mode
It is assumed that the server is run inside a Docker container and deployed to
a Kubernetes cluster. Use the Dockerfile to build the image or directly use a
[pre-built
image](https://gitlab.switch.ch/memoriav/memobase-2020/services/autodeploy-service/container_registry/129).
### Development mode
1. It is recommended to use a
[virtualenv](https://virtualenv.pypa.io/en/latest/). Inside the project
root folder type:
a. `virtualenv venv`
b. `source venv/bin/activate`
c. `pip install -r requirements.txt`
2. `FLASK_APP=autodeploy_service_app/wsgi.py flask run`
The development server is started on `0.0.0.0:5000` per default. To override
these settings, see `flask --help`.
## Deployment Workflow for Services
1. Gitlab notifies the service via a webhook that a new Docker image has been
successfully created
2. The service clones the repository and looks into the directory structure. There are two possibilities:
- A `k8s-manifests` directory exists inside the root folder => The service
deletes old deployments and installs the Kubernetes objects with `kubectl`
- A `helm-charts` directory exists inside the root folder => The service
deletes old Helm charts and installs the new one with `helm`
## Requirements and Conventions Regarding Services
The service makes certain assumptions about the project being deployed:
- Existence of a Helm chart in the directory `helm-charts/`.
- A Gitlab webhook for _pipeline events_ is set, pointing to the address where the
service can be reached
- The Helm chart must be published in the registry. See
https://gitlab.switch.ch/memoriav/memobase-2020/utilities/ci-templates/-/tree/master/helm-chart
for further instructions
- The CI process has been successful either for a commit in the `master`
branch or for a tagged commit. The tag must be SemVer compatible
from flask import Flask, send_from_directory, redirect
from kubernetes import config
from flask_restful import Api
from flasgger import Swagger
from autodeploy_service_app.resources.AutoDeploy import AutoDeploy
import autodeploy_service_app.configuration
import os
def create_app(test_config=None):
app = Flask(__name__)
app.config.from_object(autodeploy_service_app.configuration)
api = Api(app)
app.config['SWAGGER'] = {
'title': 'autodeploy-service',
'version': 'dev',
'uiversion': 3,
'termsOfService': 'http://memobase.ch/de/disclaimer',
'description': 'service to deploy on k8s form a gitlab webhook',
'contact': {
'name': 'UB Basel',
'url': 'https://ub.unibas.ch',
'email': 'swissbib-ub@unibas.ch'},
'favicon': '/favicon.ico'}
Swagger(app)
@app.route("/")
def home():
return redirect("/apidocs")
@app.route('/favicon.ico')
def favicon():
return send_from_directory(
os.path.join(
app.root_path,
'assets'),
'favicon.ico',
mimetype='image/vnd.microsoft.icon')
api.add_resource(AutoDeploy, '/v1/autodeploy')
# TODO : maybe take that to a configuration (development vs pod running in
# k8s cluster)
try:
# to be used when inside a kubernetes cluster
config.load_incluster_config()
except BaseException:
try:
# use .kube directory
# for local development
config.load_kube_config()
except BaseException:
app.logger.error("No kubernetes cluster defined")
return app
from flask import Flask
from autodeploy_service_app import configuration
app = Flask(__name__)
app.config.from_object(configuration)
from flask_restful import Resource
from flask import request as flaskRequest
import json
import subprocess
from re import match
from os import environ, listdir, path
from shutil import rmtree
from tempfile import mkdtemp
from autodeploy_service_app.app import app
class AutoDeploy(Resource):
def post(self):
'''
Deploy on k8s
---
tags:
- AutoDeploy
requestBody:
required: true
responses:
200:
description: Success, job report gets returned
schema:
properties:
status:
type: string
example: success/failure/ignore
enum: ['success', 'failure', 'ignore']
report:
type: string
example: 'institutionXYZ-235323B: something failed!\n
institutionXYZ-235323C: that one went totally wrong'
500:
description: Impossible to get results
schema:
properties:
error:
type: string
example: Unexpected Kafka error
'''
status = 'ignore'
body = ''
output = []
try:
body = json.loads(flaskRequest.data.decode('utf-8'))
tag = ''
branch = ''
if body['object_attributes']['tag']:
tag = body['object_attributes']['ref']
else:
branch = body['object_attributes']['ref']
branch = body['object_attributes']['ref']
if body['object_attributes']['status'] == 'success':
projectName = body['project']['path_with_namespace'].split('/')[-1]
repositoryUrl = body['project']['git_http_url']
repositoryPath = body['project']['path_with_namespace']
app.logger.info('deploy-request received from ' + repositoryUrl)
# if tag = semver: deploy on prod+stage
if match(r'^(([0-9]+)\.([0-9]+)\.([0-9]+))$', tag):
app.logger.debug(
'commit with semver-tag detected: installing on prod+stage'
)
pullChartUri = (
environ['GITLAB_REGISTRY']
+ '/'
+ repositoryPath
+ ':'
+ tag
+ '-chart'
)
msgs, status = installFromRepo(pullChartUri)
output.extend(msgs)
# deploy on test
elif branch == 'master':
app.logger.debug(
'commit on master without semver-tag detected: installing on test'
)
msgs, status = installFromDir(repositoryUrl, projectName)
output.extend(msgs)
except subprocess.CalledProcessError as ex:
msg = 'command {} failed with return code {} -- STDOUT: {} -- STDERR: {}'.format(
ex.cmd, ex.returncode, ex.stdout, ex.stderr
)
output.append(msg)
status = 'failure'
app.logger.warning(msg)
except json.JSONDecodeError as ex:
msg = 'the json in the body could not be decoded: {}'.format(str(ex))
output.append(msg)
status = 'failure'
app.logger.warning(msg)
except Exception as ex:
msg = 'an exception occured: {}'.format(str(ex))
output.append(msg)
status = 'failure'
app.logger.warning(msg)
returnVal = {'status': status, 'log': output, 'body': body}
if status == 'success':
app.logger.info(json.dumps(returnVal))
return returnVal, 200
elif status == 'ignore':
app.logger.debug(json.dumps(returnVal))
return returnVal, 500
else:
app.logger.info(json.dumps(returnVal))
return returnVal, 500
def _upgrade_installation(chartsDir, projectName, filenameBase, filename=None):
app.logger.debug(
'upgrading new deployment for {}{}'.format(
projectName, '' if filename is None else ': ' + filename
)
)
try:
cmd = 'helm upgrade -i {}-deployment {}{}'.format(
filenameBase,
''
if filename is None
else '-f {} '.format(path.join(chartsDir, 'helm-values', filename)),
chartsDir,
)
proc = subprocess.run(
cmd, shell=True, capture_output=True, text=True, check=True
)
app.logger.info(
'successfully installed helm chart{}'.format(
'' if filename is None else ' with ' + filename
)
)
return (
'installing {}-deployment\n{}\n{}'.format(
filenameBase, proc.stdout, proc.stderr
),
'success',
)
except subprocess.CalledProcessError as ex:
msg = 'upgrading helm chart failed with return code {} -- STDOUT: {} -- STDERR: {}'.format(
ex.returncode, ex.stdout, ex.stderr
)
app.logger.warning(msg)
return msg, 'failure'
def installFromRepo(pullChartUri):
output = []
status = ''
app.logger.debug('pulling helm charts')
try:
proc = subprocess.run(
'helm chart pull ' + pullChartUri,
shell=True,
capture_output=True,
text=True,
check=True,
)
output.append(
'pulling charts from {}: {} (stderr: {})'.format(
pullChartUri, proc.stdout, proc.stderr
)
)
except subprocess.CalledProcessError as ex:
msg = 'pulling helm chart failed with return code {} -- STDOUT: {} -- STDERR: {}'.format(
ex.returncode, ex.stdout, ex.stderr
)
app.logger.warning(msg)
return output, 'failure'
pulledChartsDir = mkdtemp()
app.logger.debug('exporting helm charts')
try:
proc = subprocess.run(
'helm chart export ' + pullChartUri + ' -d ' + pulledChartsDir,
shell=True,
capture_output=True,
text=True,
check=True,
)
output.append(
'exporting charts to local directory: {} (stderr: {})'.format(
proc.stdout, proc.stderr
)
)
except subprocess.CalledProcessError as ex:
msg = 'exporting helm chart failed with return code {} -- STDOUT: {} -- STDERR: {}'.format(
ex.returncode, ex.stdout, ex.stderr
)
app.logger.warning(msg)
return output, 'failure'
projectName = listdir(pulledChartsDir)[0]
helmChartDir = path.join(pulledChartsDir, projectName)
helmValuesDir = path.join(helmChartDir, 'helm-values')
if path.exists(helmValuesDir):
app.logger.debug('helm value files detected')
valuesFound = False
for filename in listdir(helmValuesDir):
filenameBase = path.splitext(filename)[0]
if '-prod.' in filename and status != 'failure':
app.logger.info(
'upgrading helm chart {} for prod environment'.format(projectName)
)
msg, status = _upgrade_installation(
helmChartDir, projectName, filenameBase, filename
)
output.append(msg)
valuesFound = True
elif '-stage.' in filename and status != 'failure':
app.logger.info(
'upgrading helm chart {} for stage environment'.format(projectName)
)
msg, status = _upgrade_installation(
helmChartDir, projectName, filenameBase, filename
)
output.append(msg)
valuesFound = True
if not valuesFound:
msg = 'no helm chart for prod or stage environment found'
app.logger.info(msg)
output.append(msg)
status = 'ignore'
else:
msg = 'Could not find path to exported helm chart'
app.logger.warn(msg)
output.append(msg)
status = 'failure'
rmtree(pulledChartsDir)
try:
proc = subprocess.run(
'helm chart remove {}'.format(pullChartUri),
shell=True,
capture_output=True,
text=True,
check=True,
)
except subprocess.CalledProcessError as ex:
msg = (
'removing helm chart from local repository'
+ 'failed with return code {} -- STDOUT: {} -- STDERR: {}'.format(
ex.returncode, ex.stdout, ex.stderr
)
)
app.logger.warning(msg)
return output, status
def installFromDir(repositoryUrl, projectName):
output = []
status = ''
app.logger.debug('cloning repository')
repoDir = mkdtemp()
try:
subprocess.run(
'git clone {} {}'.format(repositoryUrl, repoDir),
shell=True,
capture_output=True,
text=True,
check=True,
)
output.append('git clone {}'.format(repositoryUrl))
except subprocess.CalledProcessError as ex:
msg = 'cloning git repo failed with return code {} -- STDOUT: {} -- STDERR: {}'.format(
ex.returncode, ex.stdout, ex.stderr
)
app.logger.warning(msg)
return output, 'failure'
helmChartDir = path.join(repoDir, 'helm-charts')
if not path.exists(helmChartDir):
msg = 'No helm-charts directory exists'
app.logger.warn(msg)
output.append(msg)
return output, 'failure'
helmValueDir = path.join(helmChartDir, 'helm-values')
if path.exists(helmValueDir):
app.logger.debug('helm value files detected')
for filename in listdir(helmValueDir):
filenameBase = path.splitext(filename)[0]
if '-test.' in filename:
app.logger.info(
'upgrading helm chart {} for test environment'.format(projectName)
)
msg, status = _upgrade_installation(
helmChartDir, projectName, filenameBase, filename
)
output.append(msg)
else:
# this section should be obsolete since we shouldn't have helm-charts w/o valuefiles
app.logger.warn('no helm value files detected. this is DEPRECATED')
msg, status = _upgrade_installation(helmChartDir, projectName, projectName)
output.append(msg)
app.logger.debug('Removing temp dir')
rmtree(repoDir)
return output, status
NAMESPACE = "memobase"
NAMESPACE = 'memobase'
# Copyright (C) 2019 The Android Open Source Project
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
# https://gerrit.googlesource.com/k8s-gerrit/+/refs/heads/master/tests/helpers/helm.py
import json
import subprocess
class Helm:
def _exec_command(self, cmd, fail_on_err=True):
base_cmd = [
"helm",
]
# for debug print (' '.join(base_cmd+cmd))
return subprocess.run(
base_cmd + cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE,
check=fail_on_err,
text=True
)
def install(
self,
chart,
name,
values_file=None,
set_values=None,
namespace=None,
fail_on_err=True,
wait=True,
):
"""Installs a chart on the cluster
Arguments:
chart {str} -- Release name or path of a helm chart
name {str} -- Name with which the chart will be installed on the cluster
Keyword Arguments:
values_file {str} -- Path to a custom values.yaml file (default: {None})
set_values {dict} -- Dictionary containing key-value-pairs that are used
to overwrite values in the values.yaml-file.
(default: {None})
namespace {str} -- Namespace to install the release into (default: {default})
fail_on_err {bool} -- Whether to fail with an exception if the installation
fails (default: {True})
wait {bool} -- Whether to wait for all pods to be ready (default: {True})
Returns:
CompletedProcess -- CompletedProcess-object returned by subprocess
containing details about the result and output of the
executed command.
"""
helm_cmd = ["install", name, chart]
if values_file:
helm_cmd.extend(("-f", values_file))
if set_values:
opt_list = ["%s=%s" % (k, v) for k, v in set_values.items()]
helm_cmd.extend(("--set", ",".join(opt_list)))
if namespace:
helm_cmd.extend(("--namespace", namespace))
if wait:
helm_cmd.append("--wait")
return self._exec_command(helm_cmd, fail_on_err)
def list(self):
"""Lists helm charts installed on the cluster.
Returns:
list -- List of helm chart realeases installed on the cluster.
"""
helm_cmd = ["list", "--all", "--output", "json"]
output = self._exec_command(helm_cmd).stdout
output = json.loads(output)
return output["Releases"]
def upgrade(
self,
chart,
name,
values_file=None,
set_values=None,
reuse_values=True,
recreate_pods=False,
fail_on_err=True,
):
"""Updates a chart on the cluster
Arguments:
chart {str} -- Release name or path of a helm chart
name {str} -- Name with which the chart will be installed on the cluster
Keyword Arguments:
values_file {str} -- Path to a custom values.yaml file (default: {None})
set_values {dict} -- Dictionary containing key-value-pairs that are used
to overwrite values in the values.yaml-file.
(default: {None})
reuse_values {bool} -- Whether to reuse existing not overwritten values
(default: {True})
recreate_pods {bool} -- Whether to restart changed pods (default: {False})
fail_on_err {bool} -- Whether to fail with an exception if the installation
fails (default: {True})
Returns:
CompletedProcess -- CompletedProcess-object returned by subprocess
containing details about the result and output of the
executed command.
"""
helm_cmd = ["upgrade", name, chart, "--wait"]
if values_file:
helm_cmd.extend(("-f", values_file))
if reuse_values:
helm_cmd.append("--reuse-values")
if recreate_pods:
helm_cmd.append("--recreate-pods")
if set_values:
opt_list = ["%s=%s" % (k, v) for k, v in set_values.items()]
helm_cmd.extend(("--set", ",".join(opt_list)))
return self._exec_command(helm_cmd, fail_on_err)
def uninstall(self, name, namespace=None, fail_on_err=True):
"""Uninstall a chart from the cluster
Arguments:
name {str} -- Name of the chart to delete
Keyword Arguments:
namespace {str} -- Namespace to uninstallnstall the release into (default: {default})
fail_on_err {bool} -- Whether to fail with an exception if the installation
fails (default: {True})
Returns:
CompletedProcess -- CompletedProcess-object returned by subprocess
containing details about the result and output of the
executed command.
"""
helm_cmd = ["uninstall", name]
if namespace:
helm_cmd.extend(("--namespace", namespace))
return self._exec_command(helm_cmd, fail_on_err)
def uninstall_all(self, exceptions=None):
"""Deletes all charts on the cluster
Keyword Arguments:
exceptions {list} -- List of chart names not to delete (default: {None})
"""
charts = self.list()
for chart in charts:
if chart["Name"] in exceptions:
continue
self.uninstall(chart["Name"])
import logging
import os
from autodeploy_service_app import create_app
if __name__ == "__main__":