Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
memoriav
Memobase 2020
services
Elasticsearch Services
Search Doc Service
Commits
ad271354
Commit
ad271354
authored
Nov 23, 2020
by
Jonas Waeber
Browse files
Upgraded lib dependency to 2.0.0
parent
8e6ce5bb
Changes
20
Show whitespace changes
Inline
Side-by-side
build.gradle
View file @
ad271354
...
...
@@ -34,7 +34,7 @@ ext {
dependencies
{
compile
group:
'org.elasticsearch.client'
,
name:
'elasticsearch-rest-high-level-client'
,
version:
'7.6.1'
implementation
'org.memobase:memobase-service-utilities:
0.14.2
'
implementation
'org.memobase:memobase-service-utilities:
2.0.0
'
// Logging Framework
implementation
"org.apache.logging.log4j:log4j-api:${log4jV}"
...
...
src/main/kotlin/DocumentsSearchDocBuilder.kt
View file @
ad271354
...
...
@@ -18,6 +18,7 @@
package
org.memobase
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonObject
import
org.apache.logging.log4j.LogManager
import
org.memobase.builders.*
...
...
@@ -26,7 +27,6 @@ import org.memobase.model.DocumentsSearchDoc
import
org.memobase.model.EnrichedDigitalMetadata
import
org.memobase.model.LanguageContainer
import
org.memobase.model.Schema
import
org.memobase.rdf.NS
class
DocumentsSearchDocBuilder
(
private
val
mediaUrl
:
String
)
{
private
val
log
=
LogManager
.
getLogger
(
"SearchDocTransform"
)
...
...
src/main/kotlin/InstitutionSearchDocBuilder.kt
View file @
ad271354
...
...
@@ -18,6 +18,7 @@
package
org.memobase
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonArray
import
com.beust.klaxon.JsonObject
import
java.util.Properties
...
...
@@ -31,7 +32,7 @@ import org.memobase.model.FacetContainer
import
org.memobase.model.InstitutionSearchDoc
import
org.memobase.model.LanguageContainer
import
org.memobase.model.Schema
import
org.memobase.rdf.NS
class
InstitutionSearchDocBuilder
(
path
:
String
,
appSettings
:
Properties
)
{
private
val
log
=
LogManager
.
getLogger
(
"InstitutionSearchDocBuilder"
)
...
...
src/main/kotlin/KafkaTopology.kt
View file @
ad271354
...
...
@@ -18,8 +18,12 @@
package
org.memobase
import
ch.memobase.reporting.Report
import
ch.memobase.reporting.ReportStatus
import
ch.memobase.settings.SettingsLoader
import
com.fasterxml.jackson.databind.ObjectMapper
import
com.fasterxml.jackson.module.kotlin.registerKotlinModule
import
java.io.StringWriter
import
org.apache.kafka.streams.StreamsBuilder
import
org.apache.kafka.streams.Topology
import
org.apache.kafka.streams.kstream.KStream
...
...
@@ -28,12 +32,8 @@ import org.apache.logging.log4j.LogManager
import
org.memobase.helpers.Default
import
org.memobase.helpers.JSON
import
org.memobase.helpers.KEYS
import
org.memobase.helpers.ReportStatus
import
org.memobase.model.DocumentsSearchDoc
import
org.memobase.model.Report
import
org.memobase.model.Schema
import
org.memobase.settings.SettingsLoader
import
java.io.StringWriter
class
KafkaTopology
(
private
val
settings
:
SettingsLoader
)
{
private
val
log
=
LogManager
.
getLogger
(
"SearchDocService"
)
...
...
@@ -66,10 +66,10 @@ class KafkaTopology(private val settings: SettingsLoader) {
try
{
Pair
(
searchDocTransform
.
transform
(
value
),
Report
(
readOnlyKey
,
"SUCCESS"
,
"Transformed message into search doc."
)
Report
(
readOnlyKey
,
ReportStatus
.
success
,
"Transformed message into search doc."
,
Service
.
name
)
)
}
catch
(
ex
:
InvalidInputException
)
{
Pair
(
DocumentsSearchDoc
.
DEFAULT
,
Report
(
readOnlyKey
,
"FAILURE"
,
ex
.
localizedMessage
))
Pair
(
DocumentsSearchDoc
.
DEFAULT
,
Report
(
readOnlyKey
,
ReportStatus
.
warning
,
ex
.
localizedMessage
,
Service
.
name
))
}
}
...
...
@@ -80,10 +80,14 @@ class KafkaTopology(private val settings: SettingsLoader) {
try
{
Pair
(
institutionSearchDoc
.
transform
(
readOnlyKey
,
value
),
Report
(
readOnlyKey
,
"SUCCESS"
,
"Transformed message into search doc."
)
Report
(
readOnlyKey
,
ReportStatus
.
success
,
"Transformed message into search doc."
,
Service
.
name
)
)
}
catch
(
ex
:
InvalidInputException
)
{
Pair
(
Default
.
institutionSearchDoc
,
Report
(
readOnlyKey
,
"FAILURE"
,
ex
.
localizedMessage
))
Pair
(
Default
.
institutionSearchDoc
,
Report
(
readOnlyKey
,
ReportStatus
.
warning
,
ex
.
localizedMessage
,
Service
.
name
))
}
}
outputStreams
(
institutionStream
)
...
...
@@ -93,7 +97,8 @@ class KafkaTopology(private val settings: SettingsLoader) {
Report
(
readOnlyKey
,
ReportStatus
.
fatal
,
"No record, memobase institution or record set present in input data: $value."
"No record, memobase institution or record set present in input data: $value."
,
Service
.
name
)
}
.
to
(
reportTopic
)
...
...
src/main/kotlin/Service.kt
View file @
ad271354
...
...
@@ -18,13 +18,16 @@
package
org.memobase
import
ch.memobase.settings.SettingsLoader
import
kotlin.system.exitProcess
import
org.apache.kafka.streams.KafkaStreams
import
org.apache.logging.log4j.LogManager
import
org.memobase.helpers.KEYS
import
org.memobase.settings.SettingsLoader
class
Service
(
file
:
String
=
"app.yml"
)
{
companion
object
{
const
val
name
=
"search-doc-service"
}
private
val
log
=
LogManager
.
getLogger
(
"SearchDocService"
)
val
settings
=
SettingsLoader
(
...
...
src/main/kotlin/builders/AgentContainerBuilder.kt
View file @
ad271354
...
...
@@ -18,14 +18,14 @@
package
org.memobase.builders
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonObject
import
org.apache.logging.log4j.LogManager
import
org.memobase.helpers.KEYS
import
org.memobase.helpers.Extract
import
org.memobase.helpers.FacetBuildHelpers
import
org.memobase.helpers.KEYS
import
org.memobase.model.AgentWithRelationContainer
import
org.memobase.model.LanguageContainer
import
org.memobase.rdf.NS
class
AgentContainerBuilder
(
private
val
identifiers
:
List
<
String
>,
...
...
src/main/kotlin/builders/DateContainerBuilder.kt
View file @
ad271354
package
org.memobase.builders
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonArray
import
com.beust.klaxon.JsonObject
import
org.apache.logging.log4j.LogManager
import
org.memobase.helpers.KEYS
import
org.memobase.helpers.DateFacetBuildHelpers
import
org.memobase.helpers.KEYS
import
org.memobase.model.DateContainer
import
org.memobase.rdf.NS
import
java.lang.NumberFormatException
class
DateContainerBuilder
(
private
val
containedIds
:
List
<
String
>)
:
IFieldBuilder
{
private
val
log
=
LogManager
.
getLogger
(
"DateContainerBuilder"
)
...
...
src/main/kotlin/builders/EnrichedFacetContainerBuilder.kt
View file @
ad271354
...
...
@@ -19,12 +19,10 @@
package
org.memobase.builders
import
com.beust.klaxon.JsonObject
import
com.fasterxml.jackson.module.kotlin.jacksonTypeRef
import
org.memobase.helpers.Extract
import
org.memobase.helpers.KEYS
import
org.memobase.model.EnrichedFacetContainer
import
org.memobase.model.LanguageContainer
import
org.memobase.rdf.NS
/**
*
...
...
src/main/kotlin/builders/FacettedContainerBuilder.kt
View file @
ad271354
...
...
@@ -18,11 +18,11 @@
package
org.memobase.builders
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonObject
import
org.memobase.helpers.KEYS
import
org.memobase.helpers.Extract
import
org.memobase.helpers.KEYS
import
org.memobase.model.FacetContainer
import
org.memobase.rdf.NS
/**
* @param identifiers: A list of identifiers to filter on.
...
...
src/main/kotlin/builders/IFieldBuilder.kt
View file @
ad271354
...
...
@@ -15,7 +15,6 @@
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package
org.memobase.builders
import
com.beust.klaxon.JsonObject
...
...
src/main/kotlin/builders/PersonFacetBuilder.kt
View file @
ad271354
...
...
@@ -18,11 +18,11 @@
package
org.memobase.builders
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonObject
import
org.apache.logging.log4j.LogManager
import
org.memobase.helpers.KEYS
import
org.memobase.helpers.FacetBuildHelpers
import
org.memobase.
rdf.N
S
import
org.memobase.
helpers.KEY
S
class
PersonFacetBuilder
:
IFieldBuilder
{
private
val
log
=
LogManager
.
getLogger
(
"PersonFacetBuilder"
)
...
...
src/main/kotlin/builders/PlaceFacetBuilder.kt
View file @
ad271354
...
...
@@ -18,11 +18,11 @@
package
org.memobase.builders
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonObject
import
org.apache.logging.log4j.LogManager
import
org.memobase.helpers.KEYS
import
org.memobase.helpers.FacetBuildHelpers
import
org.memobase.
rdf.N
S
import
org.memobase.
helpers.KEY
S
class
PlaceFacetBuilder
:
IFieldBuilder
{
private
val
log
=
LogManager
.
getLogger
(
"PersonFacetBuilder"
)
...
...
src/main/kotlin/builders/SuggestContainerBuilder.kt
View file @
ad271354
...
...
@@ -18,11 +18,11 @@
package
org.memobase.builders
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonObject
import
org.memobase.helpers.KEYS
import
org.memobase.helpers.Extract
import
org.memobase.helpers.KEYS
import
org.memobase.model.SuggestContainer
import
org.memobase.rdf.NS
class
SuggestContainerBuilder
(
private
val
hasSubjectIds
:
List
<
String
>)
:
IFieldBuilder
{
...
...
src/main/kotlin/helpers/JSON.kt
View file @
ad271354
package
org.memobase.helpers
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonArray
import
com.beust.klaxon.JsonObject
import
com.beust.klaxon.Klaxon
import
com.beust.klaxon.KlaxonException
import
org.apache.logging.log4j.LogManager
import
org.memobase.rdf.NS
import
java.io.StringReader
import
org.apache.logging.log4j.LogManager
object
JSON
{
...
...
src/main/kotlin/helpers/Meta.kt
View file @
ad271354
package
org.memobase.helpers
import
ch.memobase.rdf.NS
import
com.beust.klaxon.JsonObject
import
org.apache.logging.log4j.LogManager
import
org.memobase.model.FacetContainer
import
org.memobase.model.LanguageContainer
import
org.memobase.rdf.NS
/**
Extraction helpers for institutions and record sets.
...
...
src/main/kotlin/helpers/ReportStatus.kt
deleted
100644 → 0
View file @
8e6ce5bb
package
org.memobase.helpers
object
ReportStatus
{
const
val
success
=
"SUCCESS"
const
val
ignored
=
"IGNORED"
const
val
warning
=
"WARNING"
const
val
fatal
=
"FATAL"
}
\ No newline at end of file
src/main/kotlin/model/Report.kt
deleted
100644 → 0
View file @
8e6ce5bb
/*
* Media Linker
* Copyright (C) 2020 Memoriav
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as published by
* the Free Software Foundation, either version 3 of the License, or
* (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
package
org.memobase.model
import
com.beust.klaxon.Klaxon
import
java.time.LocalDateTime
data class
Report
(
val
id
:
String
,
val
status
:
String
,
val
message
:
String
,
val
step
:
String
=
"search-doc-service"
,
val
timestamp
:
String
=
LocalDateTime
.
now
().
toString
()
)
{
fun
toJson
():
String
{
return
Klaxon
().
toJsonString
(
this
)
}
override
fun
equals
(
other
:
Any
?):
Boolean
{
if
(
this
===
other
)
return
true
if
(
javaClass
!=
other
?.
javaClass
)
return
false
other
as
Report
if
(
id
!=
other
.
id
)
return
false
if
(
status
!=
other
.
status
)
return
false
if
(
message
!=
other
.
message
)
return
false
if
(
step
!=
other
.
step
)
return
false
return
true
}
override
fun
hashCode
():
Int
{
var
result
=
id
.
hashCode
()
result
=
31
*
result
+
status
.
hashCode
()
result
=
31
*
result
+
message
.
hashCode
()
result
=
31
*
result
+
step
.
hashCode
()
return
result
}
}
src/test/kotlin/IntegrationTest.kt
View file @
ad271354
...
...
@@ -17,23 +17,19 @@
*/
package
org.memobase
import
com.beust.klaxon.Klaxon
import
java.io.File
import
java.nio.charset.Charset
import
java.util.stream.Stream
import
org.apache.kafka.common.serialization.StringDeserializer
import
org.apache.kafka.common.serialization.StringSerializer
import
org.apache.kafka.streams.TopologyTestDriver
import
org.apache.kafka.streams.test.ConsumerRecordFactory
import
org.apache.logging.log4j.LogManager
import
org.assertj.core.api.Assertions.assertThat
import
org.junit.jupiter.api.Test
import
org.junit.jupiter.api.TestInstance
import
org.junit.jupiter.api.assertAll
import
org.junit.jupiter.params.ParameterizedTest
import
org.junit.jupiter.params.provider.MethodSource
import
org.memobase.model.Report
import
org.memobase.params.TestParam
import
java.io.File
import
java.nio.charset.Charset
import
java.util.stream.Stream
@TestInstance
(
TestInstance
.
Lifecycle
.
PER_CLASS
)
class
IntegrationTest
{
...
...
src/test/kotlin/TestFacetBuilders.kt
View file @
ad271354
...
...
@@ -18,6 +18,7 @@
package
org.memobase
import
ch.memobase.rdf.NS
import
com.beust.klaxon.json
import
java.util.stream.Stream
import
org.assertj.core.api.Assertions
...
...
@@ -32,7 +33,6 @@ import org.memobase.helpers.DateFacetBuildHelpers
import
org.memobase.helpers.KEYS
import
org.memobase.params.PersonFacetBuilderParams
import
org.memobase.params.TestDate
import
org.memobase.rdf.NS
@TestInstance
(
TestInstance
.
Lifecycle
.
PER_CLASS
)
class
TestFacetBuilders
{
...
...
src/test/kotlin/TestInstitutionSearchDoc.kt
View file @
ad271354
package
org.memobase
import
c
om.beust.klaxon.Klaxon
import
c
h.memobase.reporting.Report
import
com.fasterxml.jackson.databind.ObjectMapper
import
com.fasterxml.jackson.module.kotlin.registerKotlinModule
import
java.io.File
import
java.nio.charset.Charset
import
org.apache.kafka.common.serialization.StringDeserializer
import
org.apache.kafka.common.serialization.StringSerializer
import
org.apache.kafka.streams.TopologyTestDriver
...
...
@@ -11,13 +13,6 @@ import org.assertj.core.api.Assertions.assertThat
import
org.junit.jupiter.api.Test
import
org.junit.jupiter.api.TestInstance
import
org.junit.jupiter.api.assertAll
import
org.memobase.model.InstitutionSearchDoc
import
org.memobase.model.Report
import
java.io.File
import
java.io.StringReader
import
java.io.StringWriter
import
java.nio.charset.Charset
import
kotlin.reflect.typeOf
@TestInstance
(
TestInstance
.
Lifecycle
.
PER_CLASS
)
class
TestInstitutionSearchDoc
{
...
...
@@ -76,7 +71,8 @@ class TestInstitutionSearchDoc {
Report
(
"https://memobase.ch/institution/RadioX"
,
"SUCCESS"
,
"Transformed message into search doc."
"Transformed message into search doc."
,
Service
.
name
)
)
}
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment