Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Import Process
Table Data Transform
Commits
ea619f11
Commit
ea619f11
authored
Jul 29, 2021
by
Jonas Waeber
Browse files
Refactor step name.
Step name can now be configured.
parent
a49a7aae
Changes
15
Hide whitespace changes
Inline
Side-by-side
helm-charts/templates/app-config.yaml
View file @
ea619f11
...
...
@@ -5,6 +5,7 @@ metadata:
namespace
:
memobase
data
:
APPLICATION_ID
:
{{
.Values.applicationId
}}
REPORTING_STEP_NAME
:
"
{{
.Values.reportingStepName
}}"
TOPIC_IN
:
{{
.Values.inputTopicName
}}
TOPIC_OUT
:
{{
.Values.outputTopicName
}}
TOPIC_PROCESS
:
{{
.Values.reportingTopicName
}}
\ No newline at end of file
helm-charts/values.yaml
View file @
ea619f11
...
...
@@ -8,6 +8,8 @@ applicationId: placeholder
kafkaConfigs
:
placeholder
sftpConfigs
:
placeholder
reportingStepName
:
02-table-data-transform
inputTopicName
:
placeholder
outputTopicName
:
placeholder
reportingTopicName
:
placeholder
\ No newline at end of file
src/main/kotlin/KafkaTopology.kt
View file @
ea619f11
...
...
@@ -31,7 +31,8 @@ import org.memobase.models.Formats
import
org.memobase.models.Message
class
KafkaTopology
(
private
val
settings
:
SettingsLoader
)
{
private
val
parser
=
TableParser
(
settings
)
private
val
step
=
settings
.
appSettings
.
getProperty
(
Service
.
reportingStepNamePropName
)
private
val
parser
=
TableParser
(
step
,
settings
)
private
val
reportingTopic
=
settings
.
processReportTopic
private
val
klaxon
=
Klaxon
()
private
val
acceptedFormats
=
listOf
(
Formats
.
csv
,
Formats
.
xlsx
,
Formats
.
tsv
,
Formats
.
xls
,
Formats
.
ods
)
...
...
@@ -78,14 +79,14 @@ class KafkaTopology(private val settings: SettingsLoader) {
readOnlyKey
,
ReportStatus
.
fatal
,
"Could not parse message from data $data."
,
Service
.
name
step
))
}
else
{
Pair
(
parsedMessage
,
Report
(
readOnlyKey
,
ReportStatus
.
success
,
""
,
Service
.
name
step
))
}
}
catch
(
ex
:
KlaxonException
)
{
...
...
@@ -93,7 +94,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
readOnlyKey
,
ReportStatus
.
fatal
,
"JSON Parser Error: ${ex.localizedMessage}."
,
Service
.
name
step
))
}
}
...
...
src/main/kotlin/Service.kt
View file @
ea619f11
...
...
@@ -25,11 +25,11 @@ class Service(file: String = "app.yml") {
private
val
log
=
LogManager
.
getLogger
(
"TableDataService"
)
companion
object
{
const
val
name
=
"table-data-transform
"
const
val
reportingStepNamePropName
=
"reportingStepName
"
}
val
settings
=
SettingsLoader
(
listOf
(),
listOf
(
reportingStepNamePropName
),
file
,
useStreamsConfig
=
true
,
readSftpSettings
=
true
...
...
src/main/kotlin/TableParser.kt
View file @
ea619f11
...
...
@@ -35,7 +35,7 @@ import org.memobase.models.Formats
import
org.memobase.models.Message
import
org.memobase.models.ResultMessage
class
TableParser
(
settings
:
SettingsLoader
)
{
class
TableParser
(
private
val
step
:
String
,
settings
:
SettingsLoader
)
{
private
val
sftpClient
:
SftpClient
=
SftpClient
(
settings
.
sftpSettings
)
private
val
invalidPropertyNameCharacters
=
listOf
(
'.'
,
':'
,
'/'
,
'+'
)
...
...
@@ -48,29 +48,42 @@ class TableParser(settings: SettingsLoader) {
}
}
catch
(
ex
:
CSVFieldNumDifferentException
)
{
listOf
(
ResultMessage
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
"Invalid CSV Input: ${ex.localizedMessage}."
,
Service
.
name
)
))
ResultMessage
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
"Invalid CSV Input: ${ex.localizedMessage}."
,
step
)
)
)
}
catch
(
ex
:
InvalidInputException
)
{
listOf
(
ResultMessage
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
"Invalid Input: ${ex.localizedMessage}. Could not process any lines."
,
Service
.
name
)
))
ResultMessage
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
"Invalid Input: ${ex.localizedMessage}. Could not process any lines."
,
step
)
)
)
}
catch
(
ex
:
IllegalArgumentException
)
{
// Sheet index does not exist
listOf
(
ResultMessage
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
"Invalid Sheet Index provided. The sheet index ${metadata.tableSheetIndex} does not exist."
,
Service
.
name
)
))
listOf
(
ResultMessage
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
"Invalid Sheet Index provided. The sheet index ${metadata.tableSheetIndex} does not exist."
,
step
)
)
)
}
}
...
...
@@ -80,12 +93,12 @@ class TableParser(settings: SettingsLoader) {
val
inputStream
=
sftpClient
.
open
(
File
(
value
.
path
))
val
reader
=
csvReader
{
this
.
quoteChar
=
'"'
this
.
delimiter
=
if
(
value
.
format
==
Formats
.
csv
)
','
else
'\t'
this
.
charset
=
Charsets
.
UTF_8
.
displayName
()
// this.skipEmptyLine = true
}.
readAll
(
inputStream
)
csvReader
{
this
.
quoteChar
=
'"'
this
.
delimiter
=
if
(
value
.
format
==
Formats
.
csv
)
','
else
'\t'
this
.
charset
=
Charsets
.
UTF_8
.
displayName
()
// this.skipEmptyLine = true
}.
readAll
(
inputStream
)
var
headerProperties
=
emptyList
<
String
>()
var
count
=
0
for
(
line
in
reader
)
{
...
...
@@ -97,12 +110,12 @@ class TableParser(settings: SettingsLoader) {
val
trimmedProperty
=
property
.
trim
()
if
(
trimmedProperty
.
isEmpty
())
{
throw
InvalidInputException
(
"Missing a property name on row $count in column ${index + 1}!"
"Missing a property name on row $count in column ${index + 1}!"
)
}
if
(
trimmedProperty
.
any
{
value
->
invalidPropertyNameCharacters
.
contains
(
value
)
})
{
throw
InvalidInputException
(
"Invalid property name $trimmedProperty on row $count in column ${index + 1}! You may not use the any of the following characters: + , . "
"Invalid property name $trimmedProperty on row $count in column ${index + 1}! You may not use the any of the following characters: + , . "
)
}
}
...
...
@@ -115,12 +128,12 @@ class TableParser(settings: SettingsLoader) {
when
(
identifierValue
)
{
""
->
{
throw
InvalidInputException
(
"The row $count has an empty identifier in column ${metadata.tableIdentifierIndex}."
"The row $count has an empty identifier in column ${metadata.tableIdentifierIndex}."
)
}
in
identifierSet
->
{
throw
InvalidInputException
(
"The row $count contains a duplicated identifier in column ${metadata.tableIdentifierIndex} with another row."
"The row $count contains a duplicated identifier in column ${metadata.tableIdentifierIndex} with another row."
)
}
else
->
{
...
...
@@ -131,29 +144,29 @@ class TableParser(settings: SettingsLoader) {
}
}
catch
(
ex
:
InvalidInputException
)
{
resultMessages
.
add
(
ResultMessage
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
ex
.
localizedMessage
,
Service
.
name
)
ResultMessage
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
ex
.
localizedMessage
,
step
)
)
)
continue
}
val
keyValueMap
=
json
{
obj
(
zip
(
headerProperties
,
line
)
zip
(
headerProperties
,
line
)
)
}
val
report
=
Report
(
identifier
,
ReportStatus
.
success
,
""
,
Service
.
name
identifier
,
ReportStatus
.
success
,
""
,
step
)
resultMessages
.
add
(
ResultMessage
(
identifier
,
keyValueMap
,
report
))
}
...
...
@@ -177,19 +190,19 @@ class TableParser(settings: SettingsLoader) {
propertiesList
.
addAll
(
row
.
map
{
cell
->
if
(
retrieveCellValue
(
cell
).
isNotEmpty
())
{
if
(
retrieveCellValue
(
cell
).
any
{
char
->
invalidPropertyNameCharacters
.
contains
(
char
)
})
{
invalidPropertyNameCharacters
.
contains
(
char
)
})
{
throw
InvalidInputException
(
"The property in cell ${cell.address} contains one or more invalid characters: $invalidPropertyNameCharacters."
"The property in cell ${cell.address} contains one or more invalid characters: $invalidPropertyNameCharacters."
)
}
else
{
retrieveCellValue
(
cell
)
}
}
else
{
throw
InvalidInputException
(
"The header index is missing a value in cell ${cell.address}"
"The header index is missing a value in cell ${cell.address}"
)
}
}.
map
{
it
.
trim
()
})
...
...
@@ -202,12 +215,12 @@ class TableParser(settings: SettingsLoader) {
when
(
val
cellValue
=
retrieveCellValue
(
cell
))
{
""
->
{
throw
InvalidInputException
(
"The row ${row.rowNum} has an empty identifier in column ${metadata.tableIdentifierIndex}."
"The row ${row.rowNum} has an empty identifier in column ${metadata.tableIdentifierIndex}."
)
}
in
identifierSet
->
{
throw
InvalidInputException
(
"The row ${row.rowNum} contains a duplicated identifier in column ${metadata.tableIdentifierIndex} with another row."
"The row ${row.rowNum} contains a duplicated identifier in column ${metadata.tableIdentifierIndex} with another row."
)
}
else
->
{
...
...
@@ -217,36 +230,36 @@ class TableParser(settings: SettingsLoader) {
}
}
else
{
throw
InvalidInputException
(
"No cell found in row ${row.rowNum} for column ${metadata.tableIdentifierIndex}."
"No cell found in row ${row.rowNum} for column ${metadata.tableIdentifierIndex}."
)
}
}
}
catch
(
ex
:
InvalidInputException
)
{
return
@map
ResultMessage
(
key
,
null
,
Report
(
key
,
null
,
Report
(
key
,
ReportStatus
.
fatal
,
ex
.
localizedMessage
,
Service
.
name
)
step
)
)
}
val
jsonObject
=
json
{
obj
(
zip
(
propertiesList
,
retrieveCells
(
row
,
propertiesList
.
size
-
1
)
)
zip
(
propertiesList
,
retrieveCells
(
row
,
propertiesList
.
size
-
1
)
)
)
}
ResultMessage
(
rowIdentifier
,
jsonObject
,
Report
(
rowIdentifier
,
jsonObject
,
Report
(
rowIdentifier
,
ReportStatus
.
success
,
""
,
Service
.
name
)
step
)
)
}
// Empty rows create a null result. These are removed.
...
...
src/main/resources/app.yml
View file @
ea619f11
...
...
@@ -3,6 +3,8 @@ sftp:
port
:
${SFTP_PORT:?system}
user
:
${SFTP_USER:?system}
password
:
${SFTP_PASSWORD:?system}
app
:
reportingStepName
:
${REPORTING_STEP_NAME:?system}
kafka
:
streams
:
bootstrap.servers
:
${KAFKA_BOOTSTRAP_SERVERS:?system}
...
...
src/test/resources/data/2/output-report-1.json
View file @
ea619f11
...
...
@@ -2,5 +2,5 @@
"id"
:
"test-id-1"
,
"status"
:
"SUCCESS"
,
"message"
:
""
,
"step"
:
"t
able-data-transform
"
"step"
:
"t
est
"
}
\ No newline at end of file
src/test/resources/data/2/output-report-2.json
View file @
ea619f11
...
...
@@ -2,5 +2,5 @@
"id"
:
"test-id-2"
,
"status"
:
"SUCCESS"
,
"message"
:
""
,
"step"
:
"t
able-data-transform
"
"step"
:
"t
est
"
}
\ No newline at end of file
src/test/resources/data/3/output-report-1.json
View file @
ea619f11
...
...
@@ -2,5 +2,5 @@
"id"
:
"test-3.csv"
,
"status"
:
"FATAL"
,
"message"
:
"Invalid CSV Input: Fields num seems to be 6 on each row, but on 2th csv row, fields num is 5.."
,
"step"
:
"t
able-data-transform
"
"step"
:
"t
est
"
}
\ No newline at end of file
src/test/resources/data/4/output-report-1.json
View file @
ea619f11
...
...
@@ -2,5 +2,5 @@
"id"
:
"Test-1"
,
"status"
:
"SUCCESS"
,
"message"
:
""
,
"step"
:
"t
able-data-transform
"
"step"
:
"t
est
"
}
\ No newline at end of file
src/test/resources/data/4/output-report-2.json
View file @
ea619f11
...
...
@@ -2,5 +2,5 @@
"id"
:
"Test-2"
,
"status"
:
"SUCCESS"
,
"message"
:
""
,
"step"
:
"t
able-data-transform
"
"step"
:
"t
est
"
}
\ No newline at end of file
src/test/resources/test-1.yml
View file @
ea619f11
...
...
@@ -4,11 +4,7 @@ sftp:
user
:
user
password
:
password
app
:
sheet
:
0
header
:
count
:
3
line
:
3
identifier
:
1
reportingStepName
:
test
kafka
:
streams
:
bootstrap.servers
:
localhost:12345
...
...
src/test/resources/test-2.yml
View file @
ea619f11
...
...
@@ -4,11 +4,7 @@ sftp:
user
:
user
password
:
password
app
:
sheet
:
0
header
:
count
:
3
line
:
3
identifier
:
31
reportingStepName
:
test
kafka
:
streams
:
bootstrap.servers
:
localhost:12345
...
...
src/test/resources/test-3.yml
View file @
ea619f11
...
...
@@ -4,11 +4,7 @@ sftp:
user
:
user
password
:
password
app
:
sheet
:
0
header
:
count
:
3
line
:
3
identifier
:
1
reportingStepName
:
test
kafka
:
streams
:
bootstrap.servers
:
localhost:12345
...
...
src/test/resources/test-4.yml
View file @
ea619f11
...
...
@@ -4,11 +4,7 @@ sftp:
user
:
user
password
:
password
app
:
sheet
:
1
header
:
count
:
1
line
:
1
identifier
:
1
reportingStepName
:
test
kafka
:
streams
:
bootstrap.servers
:
localhost:12345
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment