Commit 8c609416 authored by Günter Hipler's avatar Günter Hipler
Browse files

more steps to include validations for sets and metadataPrefixes

preparations to include flexible configuations for these parameters
based on configMaps
some thoughts how to use these additional resorictions on query
component (for us by now ES)
parent 48b57566
Pipeline #23917 failed with stage
in 4 minutes and 34 seconds
FROM hseeberger/scala-sbt:8u252_1.3.12_2.13.2 AS build
#FROM hseeberger/scala-sbt:8u252_1.3.12_2.13.2 AS build
FROM hseeberger/scala-sbt:11.0.8_1.4.1_2.13.3 as build
ADD . /
WORKDIR /
......@@ -6,20 +7,16 @@ RUN sbt clean
RUN sbt stage
# Build deployable Docker image
FROM openjdk:8-jre-alpine
#FROM openjdk:8-jre-alpine
FROM amd64/adoptopenjdk:11-jre
COPY --from=build target/universal/stage /app/
RUN apk update
RUN apk upgrade
RUN apk add bash
#RUN apk update
#RUN apk upgrade
#RUN apk add bash
#see https://github.com/docker-library/openjdk/issues/289
RUN apk add --no-cache nss
#RUN apk add --no-cache nss
#VOLUME /updateDir
#VOLUME /deleteDir
#VOLUME /base
#RUN chown -R 497:497 /app
#USER 497:497
WORKDIR /app
CMD ./start.process.sh
......@@ -42,6 +42,7 @@ class OaiController @Inject()(cc: ControllerComponents)
resumptionToken: Option[String]): Action[AnyContent] =
Action { implicit request =>
val qP: Map[String, Seq[String]] = request.queryString
val runner: OaiRunner = new OaiQueryParameterChecker(
verb = verb,
......@@ -52,7 +53,7 @@ class OaiController @Inject()(cc: ControllerComponents)
identifier = identifier,
resumptionToken = resumptionToken,
allQueryParameter = qP,
configuration = config).checkQueryParameter
configuration = repository.oaiConfig).checkQueryParameter
val response = runner.run().createResponse.toString()
......
......@@ -23,14 +23,17 @@
package modules
import com.typesafe.config.Config
import com.typesafe.config.{Config, ConfigValue, ConfigValueFactory}
import javax.inject.{Inject, Singleton}
import org.apache.http.{Header, HttpHost}
import org.apache.http.message.BasicHeader
import org.elasticsearch.client.{RestClient, RestHighLevelClient}
import org.swissbib.memobase.oai.common.util.{OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig}
import org.swissbib.memobase.oai.common.util.{OAIMetadataPrefix, OAISet, OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig}
import play.Environment
import play.api.inject.ApplicationLifecycle
import java.util
import scala.jdk.CollectionConverters._
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.Future
......@@ -81,14 +84,34 @@ class ElasticsearchClient @Inject()(
//collection conversions
//https://stackoverflow.com/questions/8301947/what-is-the-difference-between-javaconverters-and-javaconversions-in-scala
val oaiConfigSet = config.getObjectList("oaiconfigs.sets").asScala.toSeq
val oaiSets = config.getObjectList("oaiconfigs.sets").asScala.toSeq.map(
o => {
OAISet(
spec = o.get("spec").unwrapped().toString,
name = o.get("name").unwrapped().toString,
includedSets = Option(o.getOrDefault("includedsets",null))
.map(_.unwrapped().asInstanceOf[util.ArrayList[String]].asScala.toList),
field = Option(o.getOrDefault("field",null)).map(_.unwrapped().toString),
fieldValue = Option(o.getOrDefault("value",null)).map(_.unwrapped().toString)
)
}
)
val oaiConfigPrefixes = config.getObjectList("oaiconfigs.metadataPrefix").asScala.toSeq
val oaiPrefixes = config.getObjectList("oaiconfigs.metadataPrefix").asScala.toSeq.map(
o =>
OAIMetadataPrefix (metadataPrefix = o.get("metadataPrefix").unwrapped().toString,
namespace = o.get("metadataNamespace").unwrapped().toString,
schema = o.get("schema").unwrapped().toString)
)
OaiConfig(
commonConfig,
identifyConfig,
OaiConfigSets(oaiConfigSet),
OaiConfigMetadataPrefixes(oaiConfigPrefixes) )
oaiSets.toList,
oaiPrefixes.toList )
}
......
......@@ -26,8 +26,8 @@ import com.typesafe.config.ConfigObject
case class OaiConfig(common: OaiCommonConfig,
identify: OaiIdentifyConfig,
sets: OaiConfigSets,
prefixes: OaiConfigMetadataPrefixes)
sets: List[OAISet],
prefixes: List[OAIMetadataPrefix])
......@@ -44,7 +44,11 @@ case class OaiIdentifyConfig(earliestDatestamp: String,
protocolVersion: String
)
case class OaiConfigSets(sets: Seq[ConfigObject])
case class OaiConfigSets(sets: Seq[OAISet])
case class OaiConfigMetadataPrefixes(prefixes: Seq[ConfigObject])
case class OAISet(spec:String, name: String,includedSets:Option[List[String]], field:Option[String],
fieldValue: Option[String] )
case class OaiConfigMetadataPrefixes(prefixes: Seq[OAIMetadataPrefix])
case class OAIMetadataPrefix(metadataPrefix:String, namespace:String, schema: String)
......@@ -24,7 +24,8 @@
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.util.ESResumptionTokenHelper
import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, ListRecordsExclusiveRunner, ListRecordsRunner, OaiRunner}
import org.swissbib.memobase.oai.common.validation.ListMetadataFormatsValidation.checkParameterByName
import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, ListMetadataFormatsRunner, ListRecordsExclusiveRunner, ListRecordsRunner, OaiRunner}
import scala.util.{Failure, Success}
......@@ -59,23 +60,74 @@ import scala.util.{Failure, Success}
*/
//noinspection ScalaStyle
object ListRecordsValidation extends ParameterValidationFunction {
override def apply(v1: OaiCheckedVerbWithParameter): OaiRunner =
v1 match {
//possible valid combination
case OaiCheckedVerbWithParameter(verb, Some(metadataPrefix), set, from, until, None, None, allParameter, configuration) =>
case OaiCheckedVerbWithParameter(Some(verb), Some(metadataPrefix), Some(set), from, until, None, None, allParameter, configuration) =>
//todo: check valid metadataPrefix
// check for additional illegal parameter
// check until not before from
// check until not before from - really ???
// check for valid dates
ListRecordsRunner(CheckedListRecordsParameter(verb.get,metadataPrefix, from, until, set
,allParameter))
checkParameterByName(allParameter, AllowedParameter.listRecordsAllAllowed) match {
case Seq() =>
if (!checkValidSet(set,configuration)) {
BadArgumentsErrorRunner(BadArgumentsParameter(verb,
Map("setSpec" -> s"$set no valid specification"),
allParameter))
}
else if (!checkValidMetadaPrefix(metadataPrefix,configuration)) {
BadArgumentsErrorRunner(BadArgumentsParameter(verb,
Map("metadataPrefix" -> s"$metadataPrefix no valid specification"),
allParameter))
} else {
ListRecordsRunner(CheckedListRecordsParameter(verb, metadataPrefix, from, until, Some(set)
, allParameter))
}
case setWithIllegalItems =>
BadArgumentsErrorRunner(BadArgumentsParameter(verb, setWithIllegalItems.map(key => (key, "")).toMap
, allParameter))
}
//only resumption token with nothing else is possible
case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, resumptionToken, allParameter, configuration) =>
//todo: hier falle ich hinein wenn ich "kein" metadata prefix habe aber auch kein resumption token, dann gibt es
// einen Fehler!
// dringend beheben!
case OaiCheckedVerbWithParameter(Some(verb), Some(metadataPrefix),None, from, until, None, None, allParameter
//request with no set
, configuration) =>
checkParameterByName(allParameter, AllowedParameter.listRecordsAllAllowed) match {
case Seq() =>
if (!checkValidMetadaPrefix(metadataPrefix,configuration)) {
BadArgumentsErrorRunner(BadArgumentsParameter(verb,
Map("metadataPrefix" -> s"$metadataPrefix no valid specification"),
allParameter))
} else {
ListRecordsRunner(CheckedListRecordsParameter(verb, metadataPrefix, from, until, Some(set)
, allParameter))
}
case setWithIllegalItems =>
BadArgumentsErrorRunner(BadArgumentsParameter(verb, setWithIllegalItems.map(key => (key, "")).toMap
, allParameter))
}
case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, Some(resumptionToken), allParameter, configuration) =>
ESResumptionTokenHelper.createResumptionTokenFromOaiToken(resumptionToken.get) match {
ESResumptionTokenHelper.createResumptionTokenFromOaiToken(resumptionToken) match {
case Success(rt) =>
checkParameterByName(allParameter, AllowedParameter.listRecordResumption) match {
case Seq() => ListRecordsExclusiveRunner(CheckedListRecordsParameterExclusive(verb.get, rt
......@@ -93,7 +145,7 @@ object ListRecordsValidation extends ParameterValidationFunction {
case OaiCheckedVerbWithParameter(verb, _, _, _, _, _, _, allParameter, configuration) => ???
case OaiCheckedVerbWithParameter(Some(verb), _, _, _, _, _, _, allParameter, configuration) => ???
//todo: das nochmals anschauen
......
......@@ -22,6 +22,7 @@
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.util.OaiConfig
import org.swissbib.memobase.oai.common.verb.OaiVerb
import org.swissbib.memobase.oai.common.verb.OaiVerb.OaiVerb
import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, OaiRunner}
......@@ -37,7 +38,7 @@ case class OaiCheckedVerbWithParameter(verb:Option[OaiVerb],
identifier: Option[String],
resumptionToken: Option[String],
allQueryParameter: Map[String, Seq[String]],
configuration: Configuration)
configuration: OaiConfig)
/*
class OaiQueryParameterChecker(verb: Option[String],
......@@ -114,7 +115,7 @@ class OaiQueryParameterChecker(verb: Option[String],
identifier: Option[String],
resumptionToken: Option[String],
allQueryParameter: Map[String, Seq[String]],
configuration: Configuration) {
configuration: OaiConfig) {
//noinspection ScalaStyle
def checkQueryParameter: OaiRunner = {
......
......@@ -23,6 +23,7 @@
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.util.{OAISet, OaiConfig}
import org.swissbib.memobase.oai.runner.OaiRunner
private object AllowedParameter {
......@@ -59,6 +60,12 @@ abstract class ParameterValidationFunction () extends Function1[OaiCheckedVerbWi
allowedList.toSet.diff(paramsInReq.keySet).toSeq
}
def checkValidSet(set: String, oaiConfig: OaiConfig): Boolean = oaiConfig.sets.exists(_.spec == set)
def checkValidMetadaPrefix(metadataPrefix: String, oaiConfig: OaiConfig): Boolean = oaiConfig.prefixes
.exists(_.metadataPrefix == metadataPrefix)
override def apply(v1: OaiCheckedVerbWithParameter): OaiRunner
}
......
......@@ -42,7 +42,7 @@ case class BadArgumentsResponse(config: Configuration
//todo create the body using the executed action in repository
val oaiFrame = createOaiFrame
val sets: Seq[ConfigObject] = repository.oaiConfig.sets.sets
val sets = repository.oaiConfig.sets
//val configTuples: Seq[(ConfigValue, ConfigValue)] = sets.map(f => (f.get("spec"),f.get("name")))
val verbBody: RewriteRule = new RewriteRule {
......
......@@ -24,6 +24,7 @@ package org.swissbib.memobase.oai.response
import com.typesafe.config.ConfigObject
import modules.OaiRepository
import org.swissbib.memobase.oai.common.util.OAIMetadataPrefix
import org.swissbib.memobase.oai.common.validation.OaiParameterBase
import play.api.Configuration
......@@ -42,20 +43,17 @@ case class ListMetadaFormatsResponse( config: Configuration
//todo create the body using the executed action in repository
val oaiFrame = createOaiFrame
val sets: Seq[ConfigObject] = repository.oaiConfig.prefixes.prefixes
val configTuples = sets.map(f => (f.get("metadataPrefix"),
f.get("schema"),
f.get("metadataNamespace")))
val sets: Seq[OAIMetadataPrefix] = repository.oaiConfig.prefixes
val verbBody: RewriteRule = new RewriteRule {
override def transform(n: Node): Seq[Node] = n match {
case elem: Elem if elem.label == "request" =>
elem ++ <ListMetadataFormats>
{configTuples.map(tuple =>
{sets.map(prefixDef =>
<metadataFormat>
<metadataPrefix>{tuple._1.unwrapped()}</metadataPrefix>
<schema>{tuple._2.unwrapped()}</schema>
<metadataNamespace>{tuple._2.unwrapped()}</metadataNamespace>
<metadataPrefix>{prefixDef.metadataPrefix}</metadataPrefix>
<schema>{prefixDef.schema}</schema>
<metadataNamespace>{prefixDef.namespace}</metadataNamespace>
</metadataFormat>
)}
</ListMetadataFormats>
......
......@@ -24,6 +24,7 @@ package org.swissbib.memobase.oai.response
import com.typesafe.config.{ConfigObject, ConfigValue}
import modules.OaiRepository
import org.swissbib.memobase.oai.common.util.OAISet
import org.swissbib.memobase.oai.common.validation.OaiParameterBase
import play.api.Configuration
......@@ -41,17 +42,16 @@ case class ListSetsResponse(config: Configuration
//todo create the body using the executed action in repository
val oaiFrame = createOaiFrame
val sets: Seq[ConfigObject] = repository.oaiConfig.sets.sets
val configTuples: Seq[(ConfigValue, ConfigValue)] = sets.map(f => (f.get("spec"),f.get("name")))
val sets = repository.oaiConfig.sets
val verbBody: RewriteRule = new RewriteRule {
override def transform(n: Node): Seq[Node] = n match {
case elem: Elem if elem.label == "request" =>
elem ++ <ListSets>
{configTuples.map(tuple =>
{sets.map(prefixDef =>
<set>
<setSpec>{tuple._1.unwrapped()}</setSpec>
<setName>{tuple._2.unwrapped()}</setName>
<setSpec>{prefixDef.spec}</setSpec>
<setName>{prefixDef.name}</setName>
</set>
)}
</ListSets>
......
......@@ -24,6 +24,7 @@ package org.swissbib.memobase.oai.response
import com.typesafe.config.ConfigObject
import modules.OaiRepository
import org.swissbib.memobase.oai.common.util.OAISet
import org.swissbib.memobase.oai.common.validation.{BadArgumentsParameter, SystemErrorParameter}
import play.api.Configuration
......@@ -42,8 +43,7 @@ case class SystemErrorResponse(config: Configuration
//todo create the body using the executed action in repository
val oaiFrame = createOaiFrame
val sets: Seq[ConfigObject] = repository.oaiConfig.sets.sets
//val configTuples: Seq[(ConfigValue, ConfigValue)] = sets.map(f => (f.get("spec"),f.get("name")))
//val sets: Seq[OAISet] = repository.oaiConfig.sets.sets
val verbBody: RewriteRule = new RewriteRule {
override def transform(n: Node): Seq[Node] = n match {
......
#include "oaiconfigmap/oai.conf"
#include "secureconfigmap/secure.conf"
include "oai"
include "secure"
play {
http.secret.key="Sl9aHk;Wb__x7ZblaWb4y]3;kcDPAr4>vsV[CC`;=]r1VWc=D8cSy4TBZ"
modules.enabled += "modules.ElasticsearchModule"
modules.enabled += "modules.KafkaModule"
#modules.enabled += "modules.KafkaModule"
}
# db connections = ((physical_core_count * 2) + effective_spindle_count)
......@@ -20,65 +23,5 @@ repository.dispatcher {
kafka : [
{
"source" : "defaultDB"
"server": "sb-us21.swissbib.unibas.ch:8080/solr/green"
"bla" : "lll"
},
{
"source" : "bb"
"server": "sb-us29.swissbib.unibas.ch:8080/solr/bb"
}
]
oaiconfigs: {
common: {
xsi_schemaLocation: "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
}
sets: [
{
"spec": "allEuropeana"
"name": "resources relevant and defined for Europeana"
},
{
"spec": "srftagesschau"
"name": "Tagesschausendungen SRF"
},
{
"spec": "swiradio"
"name": "Radiobestand swissinfo"
}
]
metadataPrefix: [
{
"metadataPrefix": "edm"
"schema": "to be defined"
"metadataNamespace": "to be defined"
},
{
"metadataPrefix": "rico"
"schema": "to be defined"
"metadataNamespace": "to be defined"
}
]
identify: {
repositoryName: "memobase OAI repositories"
baseURL: "https://oai.memobase.k8s.unibas.ch/"
protocolVersion: "2.0"
adminEmail: "admin@memobase.ch"
earliestDatestamp: "2021-01-01T00:00:00.000Z"
deletedRecord: "transient"
granularity: "YYYY-MM-DDThh:mm:ss.sssZ"
}
}
oaiconfigs: {
common: {
xsi_schemaLocation: "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
}
sets: [
{
"spec": "memobase"
"name": "Association of all sets defined by memobase for OAI export"
"includedsets": ["memobase:apf"]
}
{
"spec": "memobase:europeana"
"name": "Association of all sets defined by memobase for OAI export to Europeana"
"includedsets": ["memobase:apf"]
}
{
#brauchen wir das auf Bestandesebene? - dann ein includedsets
"spec": "memobase:apf"
"name": "Plans Fixes - more description provided by Daniel?"
"field": "institution"
"value": "apf"
}
]
metadataPrefix: [
{
"metadataPrefix": "edm"
"schema": "to be defined"
"metadataNamespace": "to be defined"
}
]
identify: {
repositoryName: "memobase OAI repositories"
baseURL: "https://oai.memobase.k8s.unibas.ch/"
protocolVersion: "2.0"
adminEmail: "admin@memobase.ch"
earliestDatestamp: "2021-01-01T00:00:00.000Z"
deletedRecord: "transient"
granularity: "YYYY-MM-DDThh:mm:ss.sssZ"
}
}
......@@ -29,7 +29,7 @@ play {
# A list of valid hosts (e.g. "example.com") or suffixes of valid hosts (e.g. ".example.com")
# Note that ".example.com" will match example.com and any subdomain of example.com, with or without a trailing dot.
# "." matches all domains, and "" matches an empty or nonexistent host.
allowed = ["localhost", ".k8s.unibas.ch", "127.0.0.1", "memobase.ch"]
allowed = ["192.168.99.100", "localhost", ".k8s.unibas.ch", "127.0.0.1", "memobase.ch"]
routeModifiers {
# If non empty, then requests will be checked if the route does not have this modifier. This is how we enable the
......
apiVersion: apps/v1
kind: Deployment
metadata:
namespace: memobase
#namespace: memobase
name: oai-deployment
labels:
app: oai-api
......@@ -21,12 +21,20 @@ spec:
containers:
- name: oai-api-prod-container
#image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/externalapis/oai:latest
image: guenterh/oaimemobase:0.5.3
image: guenterh/oaimemobase:latest
terminationMessagePolicy: FallbackToLogsOnError
ports:
- containerPort: 9000
name: http
protocol: TCP
imagePullPolicy: Always
volumeMounts:
- mountPath: /app/conf/oaiconfigmap
name: oaiconf
- mountPath: /app/conf/secureconfigmap
name: secureconf
env:
- name: RESPONSE_LISTLENGTH
value: "30"
......@@ -35,8 +43,21 @@ spec:
- name: ELASTICSEARCH_INDEX
value: oai-v2
- name: ELASTICSEARCH_HOSTS
value: localhost
value: 192.168.1.116
- name: ELASTICSEARCH_PORT
value: "8080"
- name: ELASTICSEARCH_CLUSTER
value: test-memobase-search-cluster
\ No newline at end of file
value: test-memobase-search-cluster
volumes:
- name: oaiconf
configMap:
name: configmap-oai-definitions
items:
- key: oai.conf
path: oai.conf
- name: secureconf
configMap:
name: configmap-oai-secure
items:
- key: secure.conf
path: secure.conf
apiVersion: v1
kind: Service
metadata:
namespace: memobase
#namespace: memobase
name: oai-service
labels:
app: oai-api
spec:
type: NodePort
ports:
- port: 9000
nodePort: 30001
selector:
app: oai-api
tier: web
clusterIP: None
\ No newline at end of file
#clusterIP: None
\ No newline at end of file
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment