Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Import Process
Table Data Transform
Commits
f1eb8c76
Commit
f1eb8c76
authored
May 20, 2020
by
Jonas Waeber
Browse files
Add excel test
parent
0e0784e7
Changes
7
Hide whitespace changes
Inline
Side-by-side
src/test/kotlin/TestParams.kt
View file @
f1eb8c76
...
...
@@ -24,5 +24,6 @@ data class TestParams(
val
inputFileName
:
String
,
val
inputKey
:
String
,
val
expectedOutputKey
:
String
,
val
expectedOutputDirectoryName
:
String
val
expectedOutputDirectoryName
:
String
,
val
processReportOutput
:
String
)
src/test/kotlin/Tests.kt
View file @
f1eb8c76
...
...
@@ -32,7 +32,6 @@ import org.junit.jupiter.api.TestInstance
import
org.junit.jupiter.params.ParameterizedTest
import
org.junit.jupiter.params.provider.MethodSource
import
org.memobase.extensions.EmbeddedSftpServer
import
org.memobase.settings.SettingsLoader
@TestInstance
(
TestInstance
.
Lifecycle
.
PER_CLASS
)
class
Tests
{
...
...
@@ -48,44 +47,44 @@ class Tests {
init
{
sftpServer
.
createDirectories
(
"/memobase/test_institution_1/test_record_set_1"
,
"/memobase/test_institution_2/test_record_set_2"
"/memobase/test_institution_2/test_record_set_2"
,
"/memobase/test_institution_3/test_record_set_3"
)
sftpServer
.
putFile
(
"/memobase/test_institution_1/test_record_set_1/brandt_metadaten.csv"
,
FileInputStream
(
"src/test/resources/sftp/brandt_metadaten.csv"
)
)
sftpServer
.
putFile
(
"/memobase/test_institution_2/test_record_set_2/bauGAZH_metadaten.csv"
,
FileInputStream
(
"src/test/resources/sftp/mapping_baugazh.csv"
)
)
sftpServer
.
putFile
(
"/memobase/test_institution_3/test_record_set_3/bauGAZH_metadaten.csv"
,
FileInputStream
(
"src/test/resources/sftp/excel_test_file.xlsx"
)
)
sftpServer
.
putFile
(
"/memobase/test_institution_1/test_record_set_1/brandt_metadaten.csv"
,
FileInputStream
(
"src/test/resources/sftp/brandt_metadaten.csv"
))
sftpServer
.
putFile
(
"/memobase/test_institution_2/test_record_set_2/bauGAZH_metadaten.csv"
,
FileInputStream
(
"src/test/resources/sftp/mapping_baugazh.csv"
))
}
@ParameterizedTest
@MethodSource
(
"testParams"
)
fun
`test
inputs`
(
params
:
TestParams
)
{
val
settingsLoader
=
SettingsLoader
(
listOf
(
"sheet"
,
"header.count"
,
"header.line"
,
"identifier"
),
params
.
settingsFileName
,
useStreamsConfig
=
true
,
readSftpSettings
=
true
)
val
testDriver
=
TopologyTestDriver
(
KafkaTopology
(
settingsLoader
).
build
(),
settingsLoader
.
kafkaStreamsSettings
)
val
service
=
Service
(
params
.
settingsFileName
)
val
testDriver
=
TopologyTestDriver
(
service
.
topology
,
service
.
settings
.
kafkaStreamsSettings
)
val
factory
=
ConsumerRecordFactory
(
StringSerializer
(),
StringSerializer
()
)
testDriver
.
pipeInput
(
factory
.
create
(
settings
Loader
.
inputTopic
,
params
.
inputKey
,
readFile
(
params
.
inputFileName
)
se
rvice
.
se
ttings
.
inputTopic
,
params
.
inputKey
,
readFile
(
params
.
inputFileName
)
)
)
var
record
=
testDriver
.
readOutput
(
settings
Loader
.
outputTopic
,
se
rvice
.
se
ttings
.
outputTopic
,
StringDeserializer
(),
StringDeserializer
()
)
assertThat
(
record
).
isNotNull
var
count
=
0
while
(
record
!=
null
)
{
count
+=
1
assertThat
(
record
)
...
...
@@ -94,9 +93,10 @@ class Tests {
.
hasFieldOrPropertyWithValue
(
"value"
,
readFile
(
params
.
expectedOutputDirectoryName
+
"/$count.json"
))
val
reportedRecord
=
testDriver
.
readOutput
(
"${settings
Loader
.outputTopic}-reporting"
,
"${se
rvice.se
ttings.outputTopic}-reporting"
,
StringDeserializer
(),
StringDeserializer
())
StringDeserializer
()
)
if
(
reportedRecord
!=
null
)
{
val
data
=
reportedRecord
.
value
()
...
...
@@ -109,11 +109,21 @@ class Tests {
}
record
=
testDriver
.
readOutput
(
settings
Loader
.
outputTopic
,
se
rvice
.
se
ttings
.
outputTopic
,
StringDeserializer
(),
StringDeserializer
()
)
}
val
processReport
=
testDriver
.
readOutput
(
service
.
settings
.
processReportTopic
,
StringDeserializer
(),
StringDeserializer
()
)
assertThat
(
processReport
)
.
isNotNull
.
hasFieldOrPropertyWithValue
(
"value"
,
params
.
processReportOutput
)
}
private
fun
testParams
()
=
Stream
.
of
(
...
...
@@ -123,7 +133,8 @@ class Tests {
"error_filter_input.json"
,
"brandt_metadaten.csv"
,
"brandt_metadaten.csv"
,
"error_filter_output"
"error_filter_output"
,
Klaxon
().
toJsonString
(
Report
(
"brandt_metadaten.csv"
,
"FAILURE"
,
"No valid file found!"
))
),
TestParams
(
"valid csv input"
,
...
...
@@ -131,7 +142,17 @@ class Tests {
"brandt_csv_import.json"
,
"brandt_metadaten.csv"
,
"AVGR13716"
,
"brandt_output"
"brandt_output"
,
Klaxon
().
toJsonString
(
Report
(
"brandt_metadaten.csv"
,
"SUCCESS"
,
"Transformed table data into 1 records."
))
),
TestParams
(
"valid xlsx input"
,
"test3.yml"
,
"excel_test_input.json"
,
"excel_test_file.xlsx"
,
"AVGR13716"
,
"excel_output"
,
Klaxon
().
toJsonString
(
Report
(
""
,
"FAILURE"
,
"message"
))
)
/*,
TestParams(
...
...
src/test/resources/data/excel_output/1.json
0 → 100644
View file @
f1eb8c76
{
"Exemplar-AVGRNr"
:
"AVGR13716"
,
"Permalink"
:
"https://www.gr.ch/Exemplare/13716"
,
"Titel-Title"
:
"Wintersport in Arosa"
,
"Titel-Beschreibung"
:
"Pferdesport; Ski alpin; Skispringen; Eishochey; Tourismus"
,
"Titel-ProduktionsjahrdesOriginals"
:
"1920, 1920-1929, genaues Datum nicht eruierbar"
,
"Titel-FilmPersonen"
:
"Brandt, Carl"
,
"Titel-Funktionen"
:
"Autor/in"
,
"Titel-Genre"
:
"Dokumentarfilm; Amateurfilm"
,
"Titel-Drehort"
:
"Arosa"
,
"Titel-Weiteres"
:
"Eisfest: teilweise identische Aufnahmen in AVGR12097
\u
201eAnkunft David Zogg
\u
201c ; Schanzenspringen auf Carmenna: teilweise identische Aufnahmen in AVGR12115
\u
201eTouristen auf dem Tschuggen
\u
201c"
,
"Titel-Stream-Url"
:
"https://s3-eu-west-1.amazonaws.com/streaming.av-portal.gr.ch/13716/AVGR13716.mov"
,
"Titel-Benutzerzugang"
:
"Intranet"
,
"Medium-Materialbezeichnung"
:
"Film"
,
"Medium-MedienFormat"
:
"35-mm-Film, Negativ und Positiv, Nitrat"
,
"Medium-Ton"
:
"stumm"
,
"Medium-Farbe"
:
"s/w getönt"
,
"Medium-Dauer"
:
"0:17:02"
,
"Medium-Bandlaenge"
:
"Vorhandene Elemente: AVGR9942: Negativ, Nitrat (CS, Z 986-172.8); AVGR9943: Positiv Nitrat (CS, Z 986-172.7); AVGR12098: Interpositiv / Marron 2366, Kopie 2016 (KBG); AVGR13715: Internegativ 2234, Kopie 2016 (KBG); AVGR13716: Positivkopie Farbe 2383, Kopie 2016, eingefärbte Sequenzen (KBG)"
}
\ No newline at end of file
src/test/resources/data/excel_output/r1.json
0 → 100644
View file @
f1eb8c76
{
"id"
:
"brandt_metadaten.csv"
,
"message"
:
"Ignored message due to previous error."
,
"status"
:
"FAILURE"
}
\ No newline at end of file
src/test/resources/data/excel_test_input.json
0 → 100644
View file @
f1eb8c76
{
"path"
:
"/memobase/test_institution_3/test_record_set_3/excel_test_file.xlsx"
,
"format"
:
"XLSX"
}
\ No newline at end of file
src/test/resources/sftp/excel_test_file.xlsx
0 → 100644
View file @
f1eb8c76
File added
src/test/resources/test3.yml
0 → 100644
View file @
f1eb8c76
id
:
jobXYZ
sftp
:
host
:
localhost
port
:
22000
user
:
user
password
:
password
app
:
sheet
:
1
header
:
count
:
3
line
:
3
identifier
:
1
kafka
:
streams
:
bootstrap.servers
:
localhost:12345
application.id
:
test-clinet-1234
topic
:
in
:
test-topic-in
out
:
test-topic-out
process
:
process-topic-id-reporting
\ No newline at end of file
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment