Skip to content

GitLab

Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

Commit f42be38b authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Switch to gunicorn deployment

parent 61a5f3df
Pipeline #33410 passed with stages
in 2 minutes and 22 seconds
Hide whitespace changes
Inline Side-by-side
Dockerfile
View file @ f42be38b
FROM python:3.8
FROM index.docker.io/library/python:3.8
ARG K8S_VERSION=v1.17.11
ARG HELM_VERSION=v3.6.3
WORKDIR /import_api_app
ADD import_api_app /import_api_app/
ADD requirements.txt /import_api_app/
WORKDIR /import_api_app/kubectl
#install kubectl so that helm can work in the docker container
RUN wget https://storage.googleapis.com/kubernetes-release/release/v1.16.2/bin/linux/amd64/kubectl && chmod +x ./kubectl
ENV PATH /import_api_app/kubectl:$PATH
WORKDIR /import_api_app
#install helm
RUN wget https://get.helm.sh/helm-v3.2.1-linux-amd64.tar.gz && tar -xvzf helm-v3.2.1-linux-amd64.tar.gz && mv linux-amd64 helm
ENV PATH /import_api_app/helm:$PATH
RUN ./pull-charts.sh
RUN pip install -r requirements.txt
WORKDIR /
ADD setup.py /
RUN pip install -e .
ENV FLASK_APP import_api_app
ENV FLASK_ENV production
EXPOSE 5000
ENTRYPOINT ["python"]
CMD ["/import_api_app/main.py"]
ENTRYPOINT ["gunicorn"]
CMD ["--bind", "0.0.0.0:5000", "wsgi:app", "--log-level", "info"]
RUN cd /usr/bin \
&& wget -q https://storage.googleapis.com/kubernetes-release/release/${K8S_VERSION}/bin/linux/amd64/kubectl \
&& chmod +x ./kubectl \
&& wget -q https://get.helm.sh/helm-${HELM_VERSION}-linux-amd64.tar.gz \
&& tar -xvzf helm-${HELM_VERSION}-linux-amd64.tar.gz \
&& mv linux-amd64/helm helm \
&& rm -rf linux-amd64
WORKDIR /
ADD setup.py /
RUN mkdir import_api_app \
&& pip install -e .
WORKDIR /import_api_app
ADD requirements.txt .
ADD import_api_app/pull-charts.sh .
RUN pip install -r requirements.txt \
&& ./pull-charts.sh
ADD import_api_app .
\ No newline at end of file
import_api_app/__init__.py
View file @ f42be38b
from flask import Flask, send_from_directory, redirect
from kubernetes import config
from flask_restful import Api
from flasgger import Swagger
from kubernetes.config import ConfigException
from import_api_app.resources.WriteJobResultsToDrupal import WriteJobResultToDrupal
from import_api_app.resources.WriteTypeReportToDrupal import WriteTypeReportToDrupal
from import_api_app.resources.UpdateInstitution import UpdateInstitution
from import_api_app.resources.ClearCache import ClearCache
from import_api_app.resources.UpdateRecordSet import UpdateRecordSet
from import_api_app.resources.ImportProcessStart import ImportProcessStart
from import_api_app.resources.FetchMappingFile import FetchMappingFile
from import_api_app.resources.DeleteRecord import DeleteRecord
from import_api_app.resources.DeleteRecordset import DeleteRecordset
from import_api_app.resources.DeleteInstitution import DeleteInstitution
from import_api_app.helpers.error import ImportApiError
import import_api_app.configuration
import os
def create_app():
app = Flask(__name__)
app.config.from_object(import_api_app.configuration)
# Based on this example :
# https://github.com/flasgger/flasgger/blob/master/examples/restful.py
api = Api(app)
try:
app.config['import-api-url'] = os.environ['IMPORT_API_URL']
app.config['kafka-broker-url'] = os.environ['KAFKA_BOOTSTRAP_SERVERS']
app.config['drupal-user'] = os.environ['DRUPAL_USERNAME']
app.config['drupal-password'] = os.environ['DRUPAL_PASSWORD']
app.config['drupal-api-url'] = os.environ['DRUPAL_API_URL']
app.config['drupal-api-key'] = os.environ['DRUPAL_API_KEY']
app.config['clear-cache-url'] = os.environ['CLEAR_CACHE_URL']
app.config['topic-configs'] = os.environ['TOPIC_CONFIGS']
app.config['topic-drupal-export'] = os.environ['TOPIC_DRUPAL_EXPORT']
app.config['sftp_host'] = os.environ['SFTP_HOST']
app.config['sftp_port'] = os.environ['SFTP_PORT']
app.config['sftp_user'] = os.environ['SFTP_USER']
app.config['sftp_password'] = os.environ['SFTP_PASSWORD']
app.config['tfv-kafka-configs'] = os.environ['TFV_KAFKA_SERVER_CONFIGS']
app.config['tfv-sftp-configs'] = os.environ['TFV_SFTP_CONFIGS']
app.config['tfv-topic-name'] = os.environ['TFV_TOPIC_NAME']
app.config['tfv-reporting-topic-name'] = os.environ['TFV_REPORTING_TOPIC_NAME']
app.config['env'] = os.environ['ENV']
app.config['SWAGGER'] = {
'title': 'Memobase Import API',
'version': 'dev',
'uiversion': 3,
'termsOfService': 'http://memobase.ch/de/disclaimer',
'description': 'API to start, stop, manage import processes for '
'memobase. Will be used in the Admin Interface (Drupal).',
'contact': {
'name': 'UB Basel',
'url': 'https://ub.unibas.ch',
'email': 'swissbib-ub@unibas.ch'},
'favicon': '/favicon.ico'}
Swagger(app)
except Exception as ex:
raise ImportApiError(str(ex))
@app.route("/")
def home():
return redirect("/apidocs")
@app.route('/favicon.ico')
def favicon():
return send_from_directory(
os.path.join(
app.root_path,
'assets'),
'favicon.ico',
mimetype='image/vnd.microsoft.icon')
api.add_resource(
ImportProcessStart,
'/v1/importprocess/<institution_id>/<record_set_id>/start'
)
api.add_resource(
FetchMappingFile,
'/v1/FetchMappingFile/<recordset_id>/<session_id>'
)
api.add_resource(WriteJobResultToDrupal, '/v1/drupal/<job_drupal_uuid>/<job_log_drupal_uuid>')
api.add_resource(WriteTypeReportToDrupal, '/v1/drupal/WriteElementReport')
api.add_resource(UpdateInstitution, '/v1/drupal/institution/<institution_drupal_uuid>')
api.add_resource(UpdateRecordSet, '/v1/drupal/recordset/<record_set_drupal_uuid>')
api.add_resource(
DeleteRecord,
'/v1/drupal/delete/record/<session_id>',
'/v1/drupal/delete/record/<session_id>/<dryrun>'
)
api.add_resource(
DeleteRecordset,
'/v1/drupal/delete/recordset/<session_id>',
'/v1/drupal/delete/recordset/<session_id>/<dryrun>'
)
api.add_resource(
DeleteInstitution,
'/v1/drupal/delete/institution/<session_id>',
'/v1/drupal/delete/institution/<session_id>/<dryrun>'
)
api.add_resource(ClearCache, '/v1/drupal/clearcache')
# TODO : maybe take that to a configuration (development vs pod running in
# k8s cluster)
try:
# to be used when inside a kubernetes cluster
config.load_incluster_config()
except ConfigException:
try:
# use .kube directory
# for local development
config.load_kube_config()
except ConfigException:
app.logger.error("No kubernetes cluster defined")
return app
import_api_app/app.py 0 → 100644
View file @ f42be38b
from flask import Flask
app = Flask(__name__)
import_api_app/main.py deleted 100644 → 0
View file @ 61a5f3df
from import_api_app import create_app
import logging
import os
if __name__ == "__main__":
numeric_level = getattr(logging, os.getenv('LOG_LEVEL').upper(), None)
if not isinstance(numeric_level, int):
raise ValueError(f'Invalid log level: {os.getenv("LOG_LEVEL")}')
logging.basicConfig(format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s',
level=numeric_level)
app = create_app()
app.run(host='0.0.0.0', debug=False)
import_api_app/pull-charts.sh
View file @ f42be38b
......@@ -4,10 +4,10 @@ export HELM_EXPERIMENTAL_OCI=1
GITLAB_REGISTRY="cr.gitlab.switch.ch"
TFV_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/text-file-validation
TFV_REGISTRY="cr.gitlab.switch.ch/memoriav/memobase-2020/services/import-process/text-file-validation"
TFV_CHART_VERSION="1.2.0-chart"
IPD_REGISTRY=cr.gitlab.switch.ch/memoriav/memobase-2020/services/deletion-components/import-process-delete
IPD_REGISTRY="cr.gitlab.switch.ch/memoriav/memobase-2020/services/deletion-components/import-process-delete"
IPD_CHART_VERSION="0.3.0-chart"
helm chart pull ${TFV_REGISTRY}:${TFV_CHART_VERSION}
......
import_api_app/resources/WriteTypeReportToDrupal.py
View file @ f42be38b
import requests
import json
from flask_restful import Resource, request
from flask_restful import current_app
from import_api_app.helpers.error import ImportApiError
import requests
import json
from requests.auth import HTTPBasicAuth
......@@ -76,7 +75,8 @@ class WriteTypeReportToDrupal(Resource):
f"Could not write report to {node_type}/{identifier}: {e.message}")
return {
'error': f"Could not write report to {node_type}/{identifier}: "
f"{e.message}"}, 500
f"{e.message}"
}, 500
def get_drupal_uuid(self, memobase_id: str, node_type: str):
url = f'{self.base_url}/jsonapi/node/{node_type}?filter[field_memobase_id]={memobase_id}'
......
import_api_app/wsgi.py 0 → 100644
View file @ f42be38b
import logging
import os
from os import environ
from flasgger import Swagger
from flask import send_from_directory, redirect
from flask.logging import default_handler
from flask_restful import Api
from kubernetes import config
from kubernetes.config import ConfigException
from import_api_app.app import app
from import_api_app.helpers.error import ImportApiError
from import_api_app.resources import ImportProcessStart, ClearCache, DeleteInstitution, \
DeleteRecordset, DeleteRecord, UpdateRecordSet, UpdateInstitution, WriteTypeReportToDrupal, \
FetchMappingFile
from import_api_app.resources.WriteJobResultsToDrupal import WriteJobResultToDrupal
try:
# to be used when inside a kubernetes cluster
config.load_incluster_config()
except ConfigException:
try:
# use .kube directory
# for local development
config.load_kube_config()
except ConfigException:
app.logger.error('No configuration for any Kubernetes Cluster detected.')
raise ImportApiError('No configuration for any Kubernetes Cluster detected.')
environ['HELM_EXPERIMENTAL_OCI'] = '1'
# If app is started via gunicorn
if __name__ != '__main__':
gunicorn_logger = logging.getLogger('gunicorn.error')
app.logger.handlers = gunicorn_logger.handlers
app.logger.setLevel(gunicorn_logger.level)
app.logger.removeHandler(default_handler)
app.logger.info('Starting production server')
api = Api(app)
try:
app.config['import-api-url'] = os.environ['IMPORT_API_URL']
app.config['kafka-broker-url'] = os.environ['KAFKA_BOOTSTRAP_SERVERS']
app.config['drupal-user'] = os.environ['DRUPAL_USERNAME']
app.config['drupal-password'] = os.environ['DRUPAL_PASSWORD']
app.config['drupal-api-url'] = os.environ['DRUPAL_API_URL']
app.config['drupal-api-key'] = os.environ['DRUPAL_API_KEY']
app.config['clear-cache-url'] = os.environ['CLEAR_CACHE_URL']
app.config['topic-configs'] = os.environ['TOPIC_CONFIGS']
app.config['topic-drupal-export'] = os.environ['TOPIC_DRUPAL_EXPORT']
app.config['sftp_host'] = os.environ['SFTP_HOST']
app.config['sftp_port'] = os.environ['SFTP_PORT']
app.config['sftp_user'] = os.environ['SFTP_USER']
app.config['sftp_password'] = os.environ['SFTP_PASSWORD']
app.config['tfv-kafka-configs'] = os.environ['TFV_KAFKA_SERVER_CONFIGS']
app.config['tfv-sftp-configs'] = os.environ['TFV_SFTP_CONFIGS']
app.config['tfv-topic-name'] = os.environ['TFV_TOPIC_NAME']
app.config['tfv-reporting-topic-name'] = os.environ['TFV_REPORTING_TOPIC_NAME']
app.config['env'] = os.environ['ENV']
app.config['SWAGGER'] = {
'title': 'Memobase Import API',
'version': 'dev',
'uiversion': 3,
'termsOfService': 'http://memobase.ch/de/disclaimer',
'description': 'API to start, stop, manage import processes for '
'memobase. Will be used in the Admin Interface (Drupal).',
'contact': {
'name': 'UB Basel',
'url': 'https://ub.unibas.ch',
'email': 'swissbib-ub@unibas.ch'},
'favicon': '/favicon.ico'}
Swagger(app)
except KeyError as ex:
raise ImportApiError(f'Environment variable {ex} is missing.')
@app.route("/")
def home():
return redirect("/apidocs")
@app.route('/favicon.ico')
def favicon():
return send_from_directory(
os.path.join(
app.root_path,
'assets'),
'favicon.ico',
mimetype='image/vnd.microsoft.icon')
api.add_resource(
ImportProcessStart,
'/v1/importprocess/<institution_id>/<record_set_id>/start'
)
api.add_resource(
FetchMappingFile,
'/v1/FetchMappingFile/<recordset_id>/<session_id>'
)
api.add_resource(WriteJobResultToDrupal, '/v1/drupal/<job_drupal_uuid>/<job_log_drupal_uuid>')
api.add_resource(WriteTypeReportToDrupal, '/v1/drupal/WriteElementReport')
api.add_resource(UpdateInstitution, '/v1/drupal/institution/<institution_drupal_uuid>')
api.add_resource(UpdateRecordSet, '/v1/drupal/recordset/<record_set_drupal_uuid>')
api.add_resource(
DeleteRecord,
'/v1/drupal/delete/record/<session_id>',
'/v1/drupal/delete/record/<session_id>/<dryrun>'
)
api.add_resource(
DeleteRecordset,
'/v1/drupal/delete/recordset/<session_id>',
'/v1/drupal/delete/recordset/<session_id>/<dryrun>'
)
api.add_resource(
DeleteInstitution,
'/v1/drupal/delete/institution/<session_id>',
'/v1/drupal/delete/institution/<session_id>/<dryrun>'
)
api.add_resource(ClearCache, '/v1/drupal/clearcache')
# If app is started with Flask
if __name__ == '__main__':
logging.basicConfig(format='%(levelname)-8s [%(filename)s:%(lineno)d] %(message)s')
app.logger.info('Starting development server')
app.run(host='0.0.0.0', port=5000, debug=True)
requirements.txt
View file @ f42be38b
......@@ -11,4 +11,5 @@ requests
kafka-python
pysftp
pyyaml
paramiko
\ No newline at end of file
paramiko
gunicorn
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment