import_process_start.py 5.25 KB
Newer Older
Jonas Waeber's avatar
Jonas Waeber committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
# Import API Service
# Copyright (C) 2020-2021 Memobase Project
#
# This program is free software: you can redistribute it and/or modify
# it under the terms of the GNU Affero General Public License as published by
# the Free Software Foundation, either version 3 of the License, or
# (at your option) any later version.
#
# This program is distributed in the hope that it will be useful,
# but WITHOUT ANY WARRANTY; without even the implied warranty of
# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
# GNU Affero General Public License for more details.
#
# You should have received a copy of the GNU Affero General Public License
# along with this program.  If not, see <https://www.gnu.org/licenses/>.
Jonas Waeber's avatar
Jonas Waeber committed
16
import hashlib
Jonas Waeber's avatar
Jonas Waeber committed
17
18
import os

19
from flasgger import swag_from
Jonas Waeber's avatar
Jonas Waeber committed
20
from flask_restful import Resource, reqparse, current_app
21

Jonas Waeber's avatar
Jonas Waeber committed
22
from import_api_app.helm import Helm
Jonas Waeber's avatar
Jonas Waeber committed
23
from import_api_app.helpers.mapping import MappingFileHandler
Jonas Waeber's avatar
Jonas Waeber committed
24

25
26

class ImportProcessStart(Resource):
Jonas Waeber's avatar
Jonas Waeber committed
27
28
29
30
31
32
33

    def __init__(self):
        self.host = current_app.config["sftp_host"]
        self.port = current_app.config["sftp_port"]
        self.user = current_app.config["sftp_user"]
        self.password = current_app.config["sftp_password"]
        self.base_path = current_app.config['sftp_base_path']
Jonas Waeber's avatar
Jonas Waeber committed
34
        self.topic = current_app.config['topic-configs']
Jonas Waeber's avatar
Jonas Waeber committed
35
36
37
38
39
40
41
42
43
44
45
46
47
        self.kafka_broker_url = current_app.config['kafka-broker-url']

        self.mapping_file_handler = MappingFileHandler(
            self.host,
            self.port,
            self.user,
            self.password,
            self.base_path,
            self.kafka_broker_url,
            self.topic,
            current_app.logger
        )

48
49
    @swag_from('ImportProcessStart.yml')
    def post(self, institution_id, record_set_id):
Matthias's avatar
Matthias committed
50
51
52
53
54
        # get parameters of request-body
        parser = reqparse.RequestParser()
        parser.add_argument('job-parameters', type=dict)
        args = parser.parse_args()
        job_parameters = args['job-parameters']
Matthias's avatar
Matthias committed
55
56
        job_parameters['institutionId'] = institution_id
        job_parameters['recordSetId'] = record_set_id
Matthias's avatar
Matthias committed
57

Matthias's avatar
Matthias committed
58
59
60
61
62
63
        if job_parameters is None:
            job_parameters = {}
        if 'drupalJobUuid' not in job_parameters:
            job_parameters['drupalJobUuid'] = 'adf48fb4-16ba-40b2-a207-8ff9d5aac657'
        if 'drupalJobLogResultUuid' not in job_parameters:
            job_parameters['drupalJobLogResultUuid'] = 'eef31a5c-7f73-4cac-b811-8ba948aef453'
Matthias's avatar
Matthias committed
64
65
66
        if 'sessionId' not in job_parameters:
            job_parameters['sessionId'] = 'drupal...'
        if 'xmlRecordTag' not in job_parameters:
67
            job_parameters['xmlRecordTag'] = 'record'
Matthias's avatar
Matthias committed
68
        if 'xmlIdentifierFieldName' not in job_parameters:
69
            job_parameters['xmlIdentifierFieldName'] = 'id'
Matthias's avatar
Matthias committed
70
        if 'isPublished' not in job_parameters:
Jonas Waeber's avatar
Jonas Waeber committed
71
            job_parameters['isPublished'] = 'false'
Matthias's avatar
Matthias committed
72

Jonas Waeber's avatar
Jonas Waeber committed
73
74
75
76
        # send mapping files to result:
        mapping_file_result = self.mapping_file_handler.process_mapping_files(record_set_id)
        if mapping_file_result['status'] == 'FATAL':
            return mapping_file_result, 500
77

Matthias's avatar
Matthias committed
78
        # start text-file-validation
79
80
81
82
83
84
85
86
87
88
89
        short_session_id = hashlib.sha1(job_parameters['sessionId'].encode("UTF-8")).hexdigest()[
                           :10]
        body = {'drupalJobUuid': job_parameters['drupalJobUuid'],
                'drupalJobLogResultUuid': job_parameters['drupalJobLogResultUuid'],
                'recordSetId': job_parameters['recordSetId'],
                'sessionId': job_parameters['sessionId'], 'shortSessionId': short_session_id,
                'institutionId': job_parameters['institutionId'],
                'isPublished': job_parameters['isPublished'],
                'kafkaConfigs': current_app.config['tfv-kafka-configs'],
                'sftpConfigs': current_app.config['tfv-sftp-configs'],
                'topicName': current_app.config['tfv-topic-name'],
90
                'reportingTopicName': current_app.config['tfv-reporting-topic-name'],
Matthias's avatar
Matthias committed
91
                'env': current_app.config['env']
92
                }
Jonas Waeber's avatar
Jonas Waeber committed
93

Matthias's avatar
Matthias committed
94
95
96
97
98
99
100
101
102
103
104
105
106
        if 'xmlRecordTag' in job_parameters:
            body['xmlRecordTag'] = job_parameters['xmlRecordTag']
        if 'xmlIdentifierFieldName' in job_parameters:
            body['xmlIdentifierFieldName'] = job_parameters['xmlIdentifierFieldName']
        if 'tableSheetIndex' in job_parameters:
            body['tableSheetIndex'] = job_parameters['tableSheetIndex']
        if 'tableHeaderCount' in job_parameters:
            body['tableHeaderCount'] = job_parameters['tableHeaderCount']
        if 'tableHeaderIndex' in job_parameters:
            body['tableHeaderIndex'] = job_parameters['tableHeaderIndex']
        if 'tableIdentifierIndex' in job_parameters:
            body['tableIdentifierIndex'] = job_parameters['tableIdentifierIndex']

Jonas Waeber's avatar
Jonas Waeber committed
107
108
109
        helm = Helm()
        response = helm.install(
            chart=os.path.join(current_app.root_path, "charts", 'text-file-validation'),
110
            name=short_session_id + '-' + record_set_id + '-validation',
Jonas Waeber's avatar
Jonas Waeber committed
111
112
113
114
115
116
117
            namespace=current_app.config['NAMESPACE'],
            set_values=body,
            fail_on_err=False
        )

        if response.stderr == '':
            return {'status': 'SUCCESS', 'message': response.stdout}, 200
Matthias's avatar
Matthias committed
118
        else:
Jonas Waeber's avatar
Jonas Waeber committed
119
            return {'status': 'FATAL', 'message': response.stderr}, 500