more steps to include validations for sets and metadataPrefixes

preparations to include flexible configuations for these parameters
based on configMaps
some thoughts how to use these additional resorictions on query
component (for us by now ES)

Dockerfile

-FROM hseeberger/scala-sbt:8u252_1.3.12_2.13.2 AS build
+#FROM hseeberger/scala-sbt:8u252_1.3.12_2.13.2 AS build
+FROM hseeberger/scala-sbt:11.0.8_1.4.1_2.13.3 as build
 
 ADD . /
 WORKDIR /
@@ -6,20 +7,16 @@ RUN sbt clean
 RUN sbt stage
 
 # Build deployable Docker image
-FROM openjdk:8-jre-alpine
+#FROM openjdk:8-jre-alpine
+FROM amd64/adoptopenjdk:11-jre
 COPY --from=build target/universal/stage /app/
 
-RUN apk update
-RUN apk upgrade
-RUN apk add bash
+#RUN apk update
+#RUN apk upgrade
+#RUN apk add bash
 #see https://github.com/docker-library/openjdk/issues/289
-RUN apk add --no-cache nss
+#RUN apk add --no-cache nss
 
-#VOLUME /updateDir
-#VOLUME /deleteDir
-#VOLUME /base
-#RUN chown -R 497:497 /app
-#USER 497:497
 
 WORKDIR /app
 CMD ./start.process.sh

app/controllers/OaiController.scala

@@ -42,6 +42,7 @@ class OaiController @Inject()(cc: ControllerComponents)
                     resumptionToken: Option[String]): Action[AnyContent] =
     Action { implicit request =>
 
+
       val qP: Map[String, Seq[String]] = request.queryString
       val runner: OaiRunner = new OaiQueryParameterChecker(
         verb = verb,
@@ -52,7 +53,7 @@ class OaiController @Inject()(cc: ControllerComponents)
         identifier = identifier,
         resumptionToken = resumptionToken,
         allQueryParameter = qP,
-        configuration = config).checkQueryParameter
+        configuration = repository.oaiConfig).checkQueryParameter
 
       val response = runner.run().createResponse.toString()
 

app/modules/ElasticsearchClient.scala

@@ -23,14 +23,17 @@
 
 package modules
 
-import com.typesafe.config.Config
+import com.typesafe.config.{Config, ConfigValue, ConfigValueFactory}
+
 import javax.inject.{Inject, Singleton}
 import org.apache.http.{Header, HttpHost}
 import org.apache.http.message.BasicHeader
 import org.elasticsearch.client.{RestClient, RestHighLevelClient}
-import org.swissbib.memobase.oai.common.util.{OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig}
+import org.swissbib.memobase.oai.common.util.{OAIMetadataPrefix, OAISet, OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig}
 import play.Environment
 import play.api.inject.ApplicationLifecycle
+
+import java.util
 import scala.jdk.CollectionConverters._
 import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.Future
@@ -81,14 +84,34 @@ class ElasticsearchClient @Inject()(
 
     //collection conversions
     //https://stackoverflow.com/questions/8301947/what-is-the-difference-between-javaconverters-and-javaconversions-in-scala
-    val oaiConfigSet = config.getObjectList("oaiconfigs.sets").asScala.toSeq
+    val oaiSets = config.getObjectList("oaiconfigs.sets").asScala.toSeq.map(
+      o => {
+        OAISet(
+        spec = o.get("spec").unwrapped().toString,
+        name = o.get("name").unwrapped().toString,
+        includedSets = Option(o.getOrDefault("includedsets",null))
+          .map(_.unwrapped().asInstanceOf[util.ArrayList[String]].asScala.toList),
+        field = Option(o.getOrDefault("field",null)).map(_.unwrapped().toString),
+        fieldValue = Option(o.getOrDefault("value",null)).map(_.unwrapped().toString)
+        )
+      }
+    )
     val oaiConfigPrefixes = config.getObjectList("oaiconfigs.metadataPrefix").asScala.toSeq
 
+    val oaiPrefixes = config.getObjectList("oaiconfigs.metadataPrefix").asScala.toSeq.map(
+      o =>
+        OAIMetadataPrefix (metadataPrefix = o.get("metadataPrefix").unwrapped().toString,
+          namespace = o.get("metadataNamespace").unwrapped().toString,
+          schema = o.get("schema").unwrapped().toString)
+    )
+
+
+
     OaiConfig(
       commonConfig,
       identifyConfig,
-      OaiConfigSets(oaiConfigSet),
-      OaiConfigMetadataPrefixes(oaiConfigPrefixes) )
+      oaiSets.toList,
+      oaiPrefixes.toList )
 
   }
 

app/org/swissbib/memobase/oai/common/util/OaiConfig.scala

@@ -26,8 +26,8 @@ import com.typesafe.config.ConfigObject
 
 case class OaiConfig(common: OaiCommonConfig,
                      identify: OaiIdentifyConfig,
-                     sets: OaiConfigSets,
-                     prefixes: OaiConfigMetadataPrefixes)
+                     sets: List[OAISet],
+                     prefixes: List[OAIMetadataPrefix])
 
 
 
@@ -44,7 +44,11 @@ case class OaiIdentifyConfig(earliestDatestamp: String,
                              protocolVersion: String
                             )
 
-case class OaiConfigSets(sets: Seq[ConfigObject])
+case class OaiConfigSets(sets: Seq[OAISet])
 
-case class OaiConfigMetadataPrefixes(prefixes: Seq[ConfigObject])
+case class OAISet(spec:String, name: String,includedSets:Option[List[String]], field:Option[String],
+                  fieldValue: Option[String] )
 
+case class OaiConfigMetadataPrefixes(prefixes: Seq[OAIMetadataPrefix])
+
+case class OAIMetadataPrefix(metadataPrefix:String, namespace:String, schema: String)

app/org/swissbib/memobase/oai/common/validation/ListRecordsValidation.scala

@@ -24,7 +24,8 @@
 package org.swissbib.memobase.oai.common.validation
 
 import org.swissbib.memobase.oai.common.util.ESResumptionTokenHelper
-import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, ListRecordsExclusiveRunner, ListRecordsRunner, OaiRunner}
+import org.swissbib.memobase.oai.common.validation.ListMetadataFormatsValidation.checkParameterByName
+import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, ListMetadataFormatsRunner, ListRecordsExclusiveRunner, ListRecordsRunner, OaiRunner}
 
 import scala.util.{Failure, Success}
 
@@ -59,23 +60,74 @@ import scala.util.{Failure, Success}
 
 */
 
+
+//noinspection ScalaStyle
 object ListRecordsValidation extends ParameterValidationFunction {
   override def apply(v1: OaiCheckedVerbWithParameter): OaiRunner =
     v1 match {
       //possible valid combination
-      case OaiCheckedVerbWithParameter(verb, Some(metadataPrefix), set, from, until, None, None, allParameter, configuration) =>
+      case OaiCheckedVerbWithParameter(Some(verb), Some(metadataPrefix), Some(set), from, until, None, None, allParameter, configuration) =>
 
-        //todo: check valid metadataPrefix
-        // check for additional illegal parameter
-        // check until not before from
+        // check until not before from - really ???
         // check for valid dates
-        ListRecordsRunner(CheckedListRecordsParameter(verb.get,metadataPrefix, from, until, set
-        ,allParameter))
+
+
+        checkParameterByName(allParameter, AllowedParameter.listRecordsAllAllowed) match {
+          case Seq() =>
+            if (!checkValidSet(set,configuration)) {
+              BadArgumentsErrorRunner(BadArgumentsParameter(verb,
+                Map("setSpec" -> s"$set no valid specification"),
+                allParameter))
+            }
+            else if (!checkValidMetadaPrefix(metadataPrefix,configuration)) {
+              BadArgumentsErrorRunner(BadArgumentsParameter(verb,
+                Map("metadataPrefix" -> s"$metadataPrefix no valid specification"),
+                allParameter))
+            } else {
+              ListRecordsRunner(CheckedListRecordsParameter(verb, metadataPrefix, from, until, Some(set)
+                , allParameter))
+
+            }
+          case setWithIllegalItems =>
+            BadArgumentsErrorRunner(BadArgumentsParameter(verb, setWithIllegalItems.map(key => (key, "")).toMap
+              , allParameter))
+        }
+
+
+
+
       //only resumption token with nothing else is possible
-      case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, resumptionToken, allParameter, configuration) =>
+
+        //todo: hier falle ich hinein wenn ich "kein" metadata prefix habe aber auch kein resumption token, dann gibt es
+        // einen Fehler!
+        // dringend beheben!
+
+      case OaiCheckedVerbWithParameter(Some(verb), Some(metadataPrefix),None, from, until, None, None, allParameter
+      //request with no set
+      , configuration) =>
+
+        checkParameterByName(allParameter, AllowedParameter.listRecordsAllAllowed) match {
+          case Seq() =>
+            if (!checkValidMetadaPrefix(metadataPrefix,configuration)) {
+              BadArgumentsErrorRunner(BadArgumentsParameter(verb,
+                Map("metadataPrefix" -> s"$metadataPrefix no valid specification"),
+                allParameter))
+            } else {
+              ListRecordsRunner(CheckedListRecordsParameter(verb, metadataPrefix, from, until, Some(set)
+                , allParameter))
+
+            }
+          case setWithIllegalItems =>
+            BadArgumentsErrorRunner(BadArgumentsParameter(verb, setWithIllegalItems.map(key => (key, "")).toMap
+              , allParameter))
+        }
+
+
+
+      case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, Some(resumptionToken), allParameter, configuration) =>
 
 
-        ESResumptionTokenHelper.createResumptionTokenFromOaiToken(resumptionToken.get) match {
+        ESResumptionTokenHelper.createResumptionTokenFromOaiToken(resumptionToken) match {
           case Success(rt) =>
             checkParameterByName(allParameter, AllowedParameter.listRecordResumption) match {
               case Seq() => ListRecordsExclusiveRunner(CheckedListRecordsParameterExclusive(verb.get, rt
@@ -93,7 +145,7 @@ object ListRecordsValidation extends ParameterValidationFunction {
 
 
 
-      case OaiCheckedVerbWithParameter(verb, _, _, _, _, _, _, allParameter, configuration) => ???
+      case OaiCheckedVerbWithParameter(Some(verb), _, _, _, _, _, _, allParameter, configuration) => ???
 
         //todo: das nochmals anschauen
 

app/org/swissbib/memobase/oai/common/validation/OaiQueryParameterChecker.scala

@@ -22,6 +22,7 @@
 
 package org.swissbib.memobase.oai.common.validation
 
+import org.swissbib.memobase.oai.common.util.OaiConfig
 import org.swissbib.memobase.oai.common.verb.OaiVerb
 import org.swissbib.memobase.oai.common.verb.OaiVerb.OaiVerb
 import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, OaiRunner}
@@ -37,7 +38,7 @@ case class OaiCheckedVerbWithParameter(verb:Option[OaiVerb],
                                 identifier: Option[String],
                                 resumptionToken: Option[String],
                                 allQueryParameter: Map[String, Seq[String]],
-                                configuration: Configuration)
+                                configuration: OaiConfig)
 
 /*
 class OaiQueryParameterChecker(verb: Option[String],
@@ -114,7 +115,7 @@ class OaiQueryParameterChecker(verb: Option[String],
                                identifier: Option[String],
                                resumptionToken: Option[String],
                                allQueryParameter: Map[String, Seq[String]],
-                               configuration: Configuration) {
+                               configuration: OaiConfig) {
   //noinspection ScalaStyle
   def checkQueryParameter: OaiRunner = {
 

app/org/swissbib/memobase/oai/common/validation/ParameterValidationFunction.scala

@@ -23,6 +23,7 @@
 
 package org.swissbib.memobase.oai.common.validation
 
+import org.swissbib.memobase.oai.common.util.{OAISet, OaiConfig}
 import org.swissbib.memobase.oai.runner.OaiRunner
 
 private object AllowedParameter {
@@ -59,6 +60,12 @@ abstract class ParameterValidationFunction () extends Function1[OaiCheckedVerbWi
     allowedList.toSet.diff(paramsInReq.keySet).toSeq
   }
 
+  def checkValidSet(set: String, oaiConfig: OaiConfig): Boolean = oaiConfig.sets.exists(_.spec == set)
+
+
+  def checkValidMetadaPrefix(metadataPrefix: String, oaiConfig: OaiConfig): Boolean = oaiConfig.prefixes
+    .exists(_.metadataPrefix == metadataPrefix)
+
 
   override def apply(v1: OaiCheckedVerbWithParameter): OaiRunner
 }

app/org/swissbib/memobase/oai/response/BadArgumentsResponse.scala

@@ -42,7 +42,7 @@ case class BadArgumentsResponse(config: Configuration
 
     //todo create the body using the executed action in repository
     val oaiFrame = createOaiFrame
-    val sets: Seq[ConfigObject] = repository.oaiConfig.sets.sets
+    val sets = repository.oaiConfig.sets
     //val configTuples: Seq[(ConfigValue, ConfigValue)] = sets.map(f => (f.get("spec"),f.get("name")))
 
     val verbBody: RewriteRule = new RewriteRule {

app/org/swissbib/memobase/oai/response/ListMetadaFormatsResponse.scala

@@ -24,6 +24,7 @@ package org.swissbib.memobase.oai.response
 
 import com.typesafe.config.ConfigObject
 import modules.OaiRepository
+import org.swissbib.memobase.oai.common.util.OAIMetadataPrefix
 import org.swissbib.memobase.oai.common.validation.OaiParameterBase
 import play.api.Configuration
 
@@ -42,20 +43,17 @@ case class ListMetadaFormatsResponse( config: Configuration
     //todo create the body using the executed action in repository
     val oaiFrame = createOaiFrame
 
-    val sets: Seq[ConfigObject] = repository.oaiConfig.prefixes.prefixes
-    val configTuples = sets.map(f => (f.get("metadataPrefix"),
-      f.get("schema"),
-      f.get("metadataNamespace")))
+    val sets: Seq[OAIMetadataPrefix] = repository.oaiConfig.prefixes
 
     val verbBody: RewriteRule = new RewriteRule {
       override def transform(n: Node): Seq[Node] = n match {
         case elem: Elem if elem.label == "request" =>
           elem ++ <ListMetadataFormats>
-            {configTuples.map(tuple =>
+            {sets.map(prefixDef =>
               <metadataFormat>
-                <metadataPrefix>{tuple._1.unwrapped()}</metadataPrefix>
-                <schema>{tuple._2.unwrapped()}</schema>
-                <metadataNamespace>{tuple._2.unwrapped()}</metadataNamespace>
+                <metadataPrefix>{prefixDef.metadataPrefix}</metadataPrefix>
+                <schema>{prefixDef.schema}</schema>
+                <metadataNamespace>{prefixDef.namespace}</metadataNamespace>
               </metadataFormat>
             )}
           </ListMetadataFormats>

app/org/swissbib/memobase/oai/response/ListSetsResponse.scala

@@ -24,6 +24,7 @@ package org.swissbib.memobase.oai.response
 
 import com.typesafe.config.{ConfigObject, ConfigValue}
 import modules.OaiRepository
+import org.swissbib.memobase.oai.common.util.OAISet
 import org.swissbib.memobase.oai.common.validation.OaiParameterBase
 import play.api.Configuration
 
@@ -41,17 +42,16 @@ case class ListSetsResponse(config: Configuration
 
     //todo create the body using the executed action in repository
     val oaiFrame = createOaiFrame
-    val sets: Seq[ConfigObject] = repository.oaiConfig.sets.sets
-    val configTuples: Seq[(ConfigValue, ConfigValue)] = sets.map(f => (f.get("spec"),f.get("name")))
+    val sets = repository.oaiConfig.sets
 
     val verbBody: RewriteRule = new RewriteRule {
       override def transform(n: Node): Seq[Node] = n match {
         case elem: Elem if elem.label == "request" =>
           elem ++ <ListSets>
-            {configTuples.map(tuple =>
+            {sets.map(prefixDef =>
               <set>
-                <setSpec>{tuple._1.unwrapped()}</setSpec>
-                <setName>{tuple._2.unwrapped()}</setName>
+                <setSpec>{prefixDef.spec}</setSpec>
+                <setName>{prefixDef.name}</setName>
               </set>
             )}
           </ListSets>

app/org/swissbib/memobase/oai/response/SystemErrorResponse.scala

@@ -24,6 +24,7 @@ package org.swissbib.memobase.oai.response
 
 import com.typesafe.config.ConfigObject
 import modules.OaiRepository
+import org.swissbib.memobase.oai.common.util.OAISet
 import org.swissbib.memobase.oai.common.validation.{BadArgumentsParameter, SystemErrorParameter}
 import play.api.Configuration
 
@@ -42,8 +43,7 @@ case class SystemErrorResponse(config: Configuration
 
     //todo create the body using the executed action in repository
     val oaiFrame = createOaiFrame
-    val sets: Seq[ConfigObject] = repository.oaiConfig.sets.sets
-    //val configTuples: Seq[(ConfigValue, ConfigValue)] = sets.map(f => (f.get("spec"),f.get("name")))
+    //val sets: Seq[OAISet] = repository.oaiConfig.sets.sets
 
     val verbBody: RewriteRule = new RewriteRule {
       override def transform(n: Node): Seq[Node] = n match {

conf/application.conf

+#include "oaiconfigmap/oai.conf"
+#include "secureconfigmap/secure.conf"
+include "oai"
 include "secure"
 
 
 play {
   http.secret.key="Sl9aHk;Wb__x7ZblaWb4y]3;kcDPAr4>vsV[CC`;=]r1VWc=D8cSy4TBZ"
   modules.enabled += "modules.ElasticsearchModule"
-  modules.enabled += "modules.KafkaModule"
+  #modules.enabled += "modules.KafkaModule"
 }
 
 # db connections = ((physical_core_count * 2) + effective_spindle_count)
@@ -20,65 +23,5 @@ repository.dispatcher {
 
 
 
-kafka : [
-  {
-    "source" : "defaultDB"
-    "server": "sb-us21.swissbib.unibas.ch:8080/solr/green"
-    "bla" : "lll"
-  },
-  {
-    "source" : "bb"
-    "server": "sb-us29.swissbib.unibas.ch:8080/solr/bb"
-  }
-]
-
-
-oaiconfigs: {
-  common: {
-    xsi_schemaLocation: "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
-
-  }
-  sets: [
-    {
-      "spec": "allEuropeana"
-      "name": "resources relevant and defined for Europeana"
-    },
-    {
-      "spec": "srftagesschau"
-      "name": "Tagesschausendungen SRF"
-
-    },
-    {
-      "spec": "swiradio"
-      "name": "Radiobestand swissinfo"
-
-    }
 
-  ]
-  metadataPrefix: [
 
-    {
-      "metadataPrefix": "edm"
-      "schema": "to be defined"
-      "metadataNamespace": "to be defined"
-    },
-    {
-
-      "metadataPrefix": "rico"
-      "schema": "to be defined"
-      "metadataNamespace": "to be defined"
-
-    }
-
-
-  ]
-  identify: {
-    repositoryName: "memobase OAI repositories"
-    baseURL: "https://oai.memobase.k8s.unibas.ch/"
-    protocolVersion: "2.0"
-    adminEmail: "admin@memobase.ch"
-    earliestDatestamp: "2021-01-01T00:00:00.000Z"
-    deletedRecord: "transient"
-    granularity: "YYYY-MM-DDThh:mm:ss.sssZ"
-  }
-}

conf/oai.conf

+oaiconfigs: {
+  common: {
+    xsi_schemaLocation: "http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
+
+  }
+  sets: [
+    {
+        "spec": "memobase"
+        "name": "Association of all sets defined by memobase for OAI export"
+        "includedsets": ["memobase:apf"]
+    }
+
+    {
+        "spec": "memobase:europeana"
+        "name": "Association of all sets defined by memobase for OAI export to Europeana"
+        "includedsets": ["memobase:apf"]
+    }
+
+    {
+        #brauchen wir das auf Bestandesebene? - dann ein includedsets
+        "spec": "memobase:apf"
+        "name": "Plans Fixes - more description provided by Daniel?"
+        "field": "institution"
+        "value": "apf"
+    }
+  ]
+  metadataPrefix: [
+    {
+      "metadataPrefix": "edm"
+      "schema": "to be defined"
+      "metadataNamespace": "to be defined"
+    }
+
+  ]
+  identify: {
+    repositoryName: "memobase OAI repositories"
+    baseURL: "https://oai.memobase.k8s.unibas.ch/"
+    protocolVersion: "2.0"
+    adminEmail: "admin@memobase.ch"
+    earliestDatestamp: "2021-01-01T00:00:00.000Z"
+    deletedRecord: "transient"
+    granularity: "YYYY-MM-DDThh:mm:ss.sssZ"
+  }
+}

conf/secure.conf

@@ -29,7 +29,7 @@ play {
       # A list of valid hosts (e.g. "example.com") or suffixes of valid hosts (e.g. ".example.com")
       # Note that ".example.com" will match example.com and any subdomain of example.com, with or without a trailing dot.
       # "." matches all domains, and "" matches an empty or nonexistent host.
-      allowed = ["localhost", ".k8s.unibas.ch", "127.0.0.1", "memobase.ch"]
+      allowed = ["192.168.99.100", "localhost", ".k8s.unibas.ch", "127.0.0.1", "memobase.ch"]
 
       routeModifiers {
         # If non empty, then requests will be checked if the route does not have this modifier. This is how we enable the

k8s-manifests/oai-prod-deploy.yaml

 apiVersion: apps/v1
 kind: Deployment
 metadata:
-  namespace: memobase
+  #namespace: memobase
   name: oai-deployment
   labels:
     app: oai-api
@@ -21,12 +21,20 @@ spec:
       containers:
         - name: oai-api-prod-container
           #image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/externalapis/oai:latest
-          image: guenterh/oaimemobase:0.5.3
+          image: guenterh/oaimemobase:latest
+          terminationMessagePolicy: FallbackToLogsOnError
           ports:
             - containerPort: 9000
               name: http
               protocol: TCP
           imagePullPolicy: Always
+
+          volumeMounts:
+            - mountPath: /app/conf/oaiconfigmap
+              name: oaiconf
+            - mountPath: /app/conf/secureconfigmap
+              name: secureconf
+
           env:
             - name: RESPONSE_LISTLENGTH
               value: "30"
@@ -35,8 +43,21 @@ spec:
             - name: ELASTICSEARCH_INDEX
               value: oai-v2
             - name: ELASTICSEARCH_HOSTS
-              value: localhost
+              value: 192.168.1.116
             - name: ELASTICSEARCH_PORT
               value: "8080"
             - name: ELASTICSEARCH_CLUSTER
-              value: test-memobase-search-cluster
\ No newline at end of file
+              value: test-memobase-search-cluster
+      volumes:
+        - name: oaiconf
+          configMap:
+            name: configmap-oai-definitions
+            items:
+              - key: oai.conf
+                path: oai.conf
+        - name: secureconf
+          configMap:
+            name: configmap-oai-secure
+            items:
+              - key: secure.conf
+                path: secure.conf

k8s-manifests/oai-prod-service.yaml

 apiVersion: v1
 kind: Service
 metadata:
-  namespace: memobase
+  #namespace: memobase
   name: oai-service
   labels:
     app: oai-api
 spec:
+  type: NodePort
   ports:
     - port: 9000
+      nodePort: 30001
   selector:
     app: oai-api
     tier: web
-  clusterIP: None
\ No newline at end of file
+  #clusterIP: None
\ No newline at end of file