Skip to content

GitLab

Commit ea619f11 authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Refactor step name. 

Step name can now be configured.
parent a49a7aae
Hide whitespace changes
Inline Side-by-side
Showing with 98 additions and 95 deletions
helm-charts/templates/app-config.yaml
View file @ ea619f11
......@@ -5,6 +5,7 @@ metadata:
namespace: memobase
data:
APPLICATION_ID: {{ .Values.applicationId }}
REPORTING_STEP_NAME: "{{ .Values.reportingStepName }}"
TOPIC_IN: {{ .Values.inputTopicName }}
TOPIC_OUT: {{ .Values.outputTopicName }}
TOPIC_PROCESS: {{ .Values.reportingTopicName }}
\ No newline at end of file
helm-charts/values.yaml
View file @ ea619f11
......@@ -8,6 +8,8 @@ applicationId: placeholder
kafkaConfigs: placeholder
sftpConfigs: placeholder
reportingStepName: 02-table-data-transform
inputTopicName: placeholder
outputTopicName: placeholder
reportingTopicName: placeholder
\ No newline at end of file
src/main/kotlin/KafkaTopology.kt
View file @ ea619f11
......@@ -31,7 +31,8 @@ import org.memobase.models.Formats
import org.memobase.models.Message
class KafkaTopology(private val settings: SettingsLoader) {
private val parser = TableParser(settings)
private val step = settings.appSettings.getProperty(Service.reportingStepNamePropName)
private val parser = TableParser(step, settings)
private val reportingTopic = settings.processReportTopic
private val klaxon = Klaxon()
private val acceptedFormats = listOf(Formats.csv, Formats.xlsx, Formats.tsv, Formats.xls, Formats.ods)
......@@ -78,14 +79,14 @@ class KafkaTopology(private val settings: SettingsLoader) {
readOnlyKey,
ReportStatus.fatal,
"Could not parse message from data $data.",
Service.name
step
))
} else {
Pair(parsedMessage, Report(
readOnlyKey,
ReportStatus.success,
"",
Service.name
step
))
}
} catch (ex: KlaxonException) {
......@@ -93,7 +94,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
readOnlyKey,
ReportStatus.fatal,
"JSON Parser Error: ${ex.localizedMessage}.",
Service.name
step
))
}
}
......
src/main/kotlin/Service.kt
View file @ ea619f11
......@@ -25,11 +25,11 @@ class Service(file: String = "app.yml") {
private val log = LogManager.getLogger("TableDataService")
companion object {
const val name = "table-data-transform"
const val reportingStepNamePropName = "reportingStepName"
}
val settings = SettingsLoader(
listOf(),
listOf(reportingStepNamePropName),
file,
useStreamsConfig = true,
readSftpSettings = true
......
src/main/kotlin/TableParser.kt
View file @ ea619f11
......@@ -35,7 +35,7 @@ import org.memobase.models.Formats
import org.memobase.models.Message
import org.memobase.models.ResultMessage
class TableParser(settings: SettingsLoader) {
class TableParser(private val step: String, settings: SettingsLoader) {
private val sftpClient: SftpClient = SftpClient(settings.sftpSettings)
private val invalidPropertyNameCharacters = listOf('.', ':', '/', '+')
......@@ -48,29 +48,42 @@ class TableParser(settings: SettingsLoader) {
}
} catch (ex: CSVFieldNumDifferentException) {
listOf(
ResultMessage(key,
null,
Report(key,
ReportStatus.fatal,
"Invalid CSV Input: ${ex.localizedMessage}.",
Service.name)
))
ResultMessage(
key,
null,
Report(
key,
ReportStatus.fatal,
"Invalid CSV Input: ${ex.localizedMessage}.",
step
)
)
)
} catch (ex: InvalidInputException) {
listOf(
ResultMessage(key,
null,
Report(key,
ReportStatus.fatal,
"Invalid Input: ${ex.localizedMessage}. Could not process any lines.",
Service.name)
))
ResultMessage(
key,
null,
Report(
key,
ReportStatus.fatal,
"Invalid Input: ${ex.localizedMessage}. Could not process any lines.",
step
)
)
)
} catch (ex: IllegalArgumentException) { // Sheet index does not exist
listOf(ResultMessage(key, null,
Report(key,
ReportStatus.fatal,
"Invalid Sheet Index provided. The sheet index ${metadata.tableSheetIndex} does not exist.",
Service.name)
))
listOf(
ResultMessage(
key, null,
Report(
key,
ReportStatus.fatal,
"Invalid Sheet Index provided. The sheet index ${metadata.tableSheetIndex} does not exist.",
step
)
)
)
}
}
......@@ -80,12 +93,12 @@ class TableParser(settings: SettingsLoader) {
val inputStream = sftpClient.open(File(value.path))
val reader =
csvReader {
this.quoteChar = '"'
this.delimiter = if (value.format == Formats.csv) ',' else '\t'
this.charset = Charsets.UTF_8.displayName()
// this.skipEmptyLine = true
}.readAll(inputStream)
csvReader {
this.quoteChar = '"'
this.delimiter = if (value.format == Formats.csv) ',' else '\t'
this.charset = Charsets.UTF_8.displayName()
// this.skipEmptyLine = true
}.readAll(inputStream)
var headerProperties = emptyList<String>()
var count = 0
for (line in reader) {
......@@ -97,12 +110,12 @@ class TableParser(settings: SettingsLoader) {
val trimmedProperty = property.trim()
if (trimmedProperty.isEmpty()) {
throw InvalidInputException(
"Missing a property name on row $count in column ${index + 1}!"
"Missing a property name on row $count in column ${index + 1}!"
)
}
if (trimmedProperty.any { value -> invalidPropertyNameCharacters.contains(value) }) {
throw InvalidInputException(
"Invalid property name $trimmedProperty on row $count in column ${index + 1}! You may not use the any of the following characters: + , . "
"Invalid property name $trimmedProperty on row $count in column ${index + 1}! You may not use the any of the following characters: + , . "
)
}
}
......@@ -115,12 +128,12 @@ class TableParser(settings: SettingsLoader) {
when (identifierValue) {
"" -> {
throw InvalidInputException(
"The row $count has an empty identifier in column ${metadata.tableIdentifierIndex}."
"The row $count has an empty identifier in column ${metadata.tableIdentifierIndex}."
)
}
in identifierSet -> {
throw InvalidInputException(
"The row $count contains a duplicated identifier in column ${metadata.tableIdentifierIndex} with another row."
"The row $count contains a duplicated identifier in column ${metadata.tableIdentifierIndex} with another row."
)
}
else -> {
......@@ -131,29 +144,29 @@ class TableParser(settings: SettingsLoader) {
}
} catch (ex: InvalidInputException) {
resultMessages.add(
ResultMessage(
key,
null,
Report(
key,
ReportStatus.fatal,
ex.localizedMessage,
Service.name
)
ResultMessage(
key,
null,
Report(
key,
ReportStatus.fatal,
ex.localizedMessage,
step
)
)
)
continue
}
val keyValueMap = json {
obj(
zip(headerProperties, line)
zip(headerProperties, line)
)
}
val report = Report(
identifier,
ReportStatus.success,
"",
Service.name
identifier,
ReportStatus.success,
"",
step
)
resultMessages.add(ResultMessage(identifier, keyValueMap, report))
}
......@@ -177,19 +190,19 @@ class TableParser(settings: SettingsLoader) {
propertiesList.addAll(row.map { cell ->
if (retrieveCellValue(cell).isNotEmpty()) {
if (retrieveCellValue(cell).any { char ->
invalidPropertyNameCharacters.contains(
char
)
}) {
invalidPropertyNameCharacters.contains(
char
)
}) {
throw InvalidInputException(
"The property in cell ${cell.address} contains one or more invalid characters: $invalidPropertyNameCharacters."
"The property in cell ${cell.address} contains one or more invalid characters: $invalidPropertyNameCharacters."
)
} else {
retrieveCellValue(cell)
}
} else {
throw InvalidInputException(
"The header index is missing a value in cell ${cell.address}"
"The header index is missing a value in cell ${cell.address}"
)
}
}.map { it.trim() })
......@@ -202,12 +215,12 @@ class TableParser(settings: SettingsLoader) {
when (val cellValue = retrieveCellValue(cell)) {
"" -> {
throw InvalidInputException(
"The row ${row.rowNum} has an empty identifier in column ${metadata.tableIdentifierIndex}."
"The row ${row.rowNum} has an empty identifier in column ${metadata.tableIdentifierIndex}."
)
}
in identifierSet -> {
throw InvalidInputException(
"The row ${row.rowNum} contains a duplicated identifier in column ${metadata.tableIdentifierIndex} with another row."
"The row ${row.rowNum} contains a duplicated identifier in column ${metadata.tableIdentifierIndex} with another row."
)
}
else -> {
......@@ -217,36 +230,36 @@ class TableParser(settings: SettingsLoader) {
}
} else {
throw InvalidInputException(
"No cell found in row ${row.rowNum} for column ${metadata.tableIdentifierIndex}."
"No cell found in row ${row.rowNum} for column ${metadata.tableIdentifierIndex}."
)
}
}
} catch (ex: InvalidInputException) {
return@map ResultMessage(
key, null, Report(
key, null, Report(
key,
ReportStatus.fatal,
ex.localizedMessage,
Service.name
)
step
)
)
}
val jsonObject = json {
obj(
zip(
propertiesList,
retrieveCells(row, propertiesList.size - 1)
)
zip(
propertiesList,
retrieveCells(row, propertiesList.size - 1)
)
)
}
ResultMessage(
rowIdentifier, jsonObject, Report(
rowIdentifier, jsonObject, Report(
rowIdentifier,
ReportStatus.success,
"",
Service.name
)
step
)
)
}
// Empty rows create a null result. These are removed.
......
src/main/resources/app.yml
View file @ ea619f11
......@@ -3,6 +3,8 @@ sftp:
port: ${SFTP_PORT:?system}
user: ${SFTP_USER:?system}
password: ${SFTP_PASSWORD:?system}
app:
reportingStepName: ${REPORTING_STEP_NAME:?system}
kafka:
streams:
bootstrap.servers: ${KAFKA_BOOTSTRAP_SERVERS:?system}
......
src/test/resources/data/2/output-report-1.json
View file @ ea619f11
......@@ -2,5 +2,5 @@
"id": "test-id-1",
"status": "SUCCESS",
"message": "",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
src/test/resources/data/2/output-report-2.json
View file @ ea619f11
......@@ -2,5 +2,5 @@
"id": "test-id-2",
"status": "SUCCESS",
"message": "",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
src/test/resources/data/3/output-report-1.json
View file @ ea619f11
......@@ -2,5 +2,5 @@
"id": "test-3.csv",
"status": "FATAL",
"message": "Invalid CSV Input: Fields num seems to be 6 on each row, but on 2th csv row, fields num is 5..",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
src/test/resources/data/4/output-report-1.json
View file @ ea619f11
......@@ -2,5 +2,5 @@
"id": "Test-1",
"status": "SUCCESS",
"message": "",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
src/test/resources/data/4/output-report-2.json
View file @ ea619f11
......@@ -2,5 +2,5 @@
"id": "Test-2",
"status": "SUCCESS",
"message": "",
"step": "table-data-transform"
"step": "test"
}
\ No newline at end of file
src/test/resources/test-1.yml
View file @ ea619f11
......@@ -4,11 +4,7 @@ sftp:
user: user
password: password
app:
sheet: 0
header:
count: 3
line: 3
identifier: 1
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
......
src/test/resources/test-2.yml
View file @ ea619f11
......@@ -4,11 +4,7 @@ sftp:
user: user
password: password
app:
sheet: 0
header:
count: 3
line: 3
identifier: 31
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
......
src/test/resources/test-3.yml
View file @ ea619f11
......@@ -4,11 +4,7 @@ sftp:
user: user
password: password
app:
sheet: 0
header:
count: 3
line: 3
identifier: 1
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
......
src/test/resources/test-4.yml
View file @ ea619f11
......@@ -4,11 +4,7 @@ sftp:
user: user
password: password
app:
sheet: 1
header:
count: 1
line: 1
identifier: 1
reportingStepName: test
kafka:
streams:
bootstrap.servers: localhost:12345
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment