Commit ea619f11 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor step name.

Step name can now be configured.
parent a49a7aae
......@@ -5,6 +5,7 @@ metadata:
namespace: memobase
data:
APPLICATION_ID: {{ .Values.applicationId }}
REPORTING_STEP_NAME: "{{ .Values.reportingStepName }}"
TOPIC_IN: {{ .Values.inputTopicName }}
TOPIC_OUT: {{ .Values.outputTopicName }}
TOPIC_PROCESS: {{ .Values.reportingTopicName }}
\ No newline at end of file
......@@ -8,6 +8,8 @@ applicationId: placeholder
kafkaConfigs: placeholder
sftpConfigs: placeholder
reportingStepName: 02-table-data-transform
inputTopicName: placeholder
outputTopicName: placeholder
reportingTopicName: placeholder
\ No newline at end of file
......@@ -31,7 +31,8 @@ import org.memobase.models.Formats
import org.memobase.models.Message
class KafkaTopology(private val settings: SettingsLoader) {
private val parser = TableParser(settings)
private val step = settings.appSettings.getProperty(Service.reportingStepNamePropName)
private val parser = TableParser(step, settings)
private val reportingTopic = settings.processReportTopic
private val klaxon = Klaxon()
private val acceptedFormats = listOf(Formats.csv, Formats.xlsx, Formats.tsv, Formats.xls, Formats.ods)
......@@ -78,14 +79,14 @@ class KafkaTopology(private val settings: SettingsLoader) {
readOnlyKey,
ReportStatus.fatal,
"Could not parse message from data $data.",
Service.name
step
))
} else {
Pair(parsedMessage, Report(
readOnlyKey,
ReportStatus.success,
"",
Service.name
step
))
}
} catch (ex: KlaxonException) {
......@@ -93,7 +94,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
readOnlyKey,
ReportStatus.fatal,
"JSON Parser Error: ${ex.localizedMessage}.",
Service.name
step
))
}
}
......
......@@ -25,11 +25,11 @@ class Service(file: String = "app.yml") {
private val log = LogManager.getLogger("TableDataService")
companion object {
const val name = "table-data-transform"
const val reportingStepNamePropName = "reportingStepName"
}
val settings = SettingsLoader(
listOf(),
listOf(reportingStepNamePropName),
file,
useStreamsConfig = true,
readSftpSettings = true
......
......@@ -35,7 +35,7 @@ import org.memobase.models.Formats
import org.memobase.models.Message
import org.memobase.models.ResultMessage
class TableParser(settings: SettingsLoader) {
class TableParser(private val step: String, settings: SettingsLoader) {
private val sftpClient: SftpClient = SftpClient(settings.sftpSettings)
private val invalidPropertyNameCharacters = listOf('.', ':', '/', '+')
......@@ -48,29 +48,42 @@ class TableParser(settings: SettingsLoader) {
}
} catch (ex: CSVFieldNumDifferentException) {
listOf(
ResultMessage(key,
ResultMessage(
key,
null,
Report(key,
Report(
key,
ReportStatus.fatal,
"Invalid CSV Input: ${ex.localizedMessage}.",
Service.name)
))
step
)
)
)
} catch (ex: InvalidInputException) {
listOf(
ResultMessage(key,
ResultMessage(
key,
null,
Report(key,
Report(
key,
ReportStatus.fatal,
"Invalid Input: ${ex.localizedMessage}. Could not process any lines.",
Service.name)
))
step
)
)
)
} catch (ex: IllegalArgumentException) { // Sheet index does not exist
listOf(ResultMessage(key, null,
Report(key,
listOf(
ResultMessage(
key, null,
Report(
key,
ReportStatus.fatal,
"Invalid Sheet Index provided. The sheet index ${metadata.tableSheetIndex} does not exist.",
Service.name)
))
step
)
)
)
}
}
......@@ -138,7 +151,7 @@ class TableParser(settings: SettingsLoader) {
key,
ReportStatus.fatal,
ex.localizedMessage,
Service.name
step
)
)
)
......@@ -153,7 +166,7 @@ class TableParser(settings: SettingsLoader) {
identifier,
ReportStatus.success,
"",
Service.name
step
)
resultMessages.add(ResultMessage(identifier, keyValueMap, report))
}
......@@ -227,7 +240,7 @@ class TableParser(settings: SettingsLoader) {
key,
ReportStatus.fatal,
ex.localizedMessage,
Service.name
step
)
)
}
......@@ -245,7 +258,7 @@ class TableParser(settings: SettingsLoader) {
rowIdentifier,
ReportStatus.success,
"",
Service.name
step
)
)
}
......
......@@ -3,6 +3,8 @@ sftp:
port: ${SFTP_PORT:?system}
user: ${SFTP_USER:?system}
password: ${SFTP_PASSWORD:?system}
app:
reportingStepName: ${REPORTING_STEP_NAME:?system}
kafka:
streams:
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:?system}
......
......@@ -2,5 +2,5 @@
"id": "test-id-1",
"status": "SUCCESS",
"message": "",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
......@@ -2,5 +2,5 @@
"id": "test-id-2",
"status": "SUCCESS",
"message": "",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
......@@ -2,5 +2,5 @@
"id": "test-3.csv",
"status": "FATAL",
"message": "Invalid CSV Input: Fields num seems to be 6 on each row, but on 2th csv row, fields num is 5..",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
......@@ -2,5 +2,5 @@
"id": "Test-1",
"status": "SUCCESS",
"message": "",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
......@@ -2,5 +2,5 @@
"id": "Test-2",
"status": "SUCCESS",
"message": "",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
......@@ -4,11 +4,7 @@ sftp:
user: user
password: password
app:
sheet: 0
header:
count: 3
line: 3
identifier: 1
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -4,11 +4,7 @@ sftp:
user: user
password: password
app:
sheet: 0
header:
count: 3
line: 3
identifier: 31
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -4,11 +4,7 @@ sftp:
user: user
password: password
app:
sheet: 0
header:
count: 3
line: 3
identifier: 1
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
......
......@@ -4,11 +4,7 @@ sftp:
user: user
password: password
app:
sheet: 1
header:
count: 1
line: 1
identifier: 1
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment