more error handling / use env variables for configuration

app/modules/ElasticsearchClient.scala

@@ -46,13 +46,16 @@ class ElasticsearchClient @Inject()(
     Future.successful(client.getOrElse(Option.empty))
   })
 
-  override val index: String = config.getString("elasticsearch.index")
+  override val index: String = getEnv("ELASTICSEARCH_INDEX")
 
   //val client: Option[RestHighLevelClient] = connect()
   override val client: Option[RestHighLevelClient] = connect()
 
   //val oai = loadOaiConfig()
 
+  override def listLength: Int = getEnv("RESPONSE_LISTLENGTH").toIntOption.getOrElse(30)
+
+  override def resumptionTokenTTL: Long = getEnv("RESUMPTION_TOKEN_TTL").toLongOption.getOrElse(3)
 
   override val oaiConfig: OaiConfig = loadOaiConfig()
   //uploadTemplates()
@@ -91,16 +94,29 @@ class ElasticsearchClient @Inject()(
 
   private def connect(): Option[RestHighLevelClient] = {
     val hosts = new ArrayBuffer[HttpHost]
-    config.getStringList("elasticsearch.hosts").forEach(
+    val configuredHosts = getEnv("ELASTICSEARCH_HOSTS").split(";")
+    val configuredPort = getEnv("ELASTICSEARCH_PORT").toIntOption.getOrElse(8080)
+    configuredHosts.foreach(
       value => {
-        val hostPort = value.split(":")
-        hosts += new HttpHost(hostPort(0), hostPort(1).toInt)
+        hosts += new HttpHost(value, configuredPort)
       }
     )
-    val headers = Array(new BasicHeader("cluster.name", config.getString("elasticsearch.cluster")).asInstanceOf[Header])
+    val headers = Array(new BasicHeader("cluster.name", getEnv("ELASTICSEARCH_CLUSTER"))
+      .asInstanceOf[Header])
     Option(new RestHighLevelClient(RestClient.builder(hosts.toArray : _*).setDefaultHeaders(headers)))
   }
 
+  private def getEnv(value: String): String = {
+    val envValue: Option[String] = sys.env.get(value)
+    if (envValue.isDefined) {
+      envValue.get
+    } else {
+      Option(System.getProperties.get(value)) match {
+        case Some(value) => value.toString
+        case None => throw new Exception("Environment variable " + value + " not available")
+      }
+    }
+  }
 
 
 }

app/modules/ElasticsearchComponent.scala

@@ -33,6 +33,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder
 import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
 import org.swissbib.memobase.oai.runner.{GetRecordFailure, ResultList}
 import play.Environment
+import utilities.Helper
+
 import scala.jdk.CollectionConverters._
 import scala.util.{Failure, Success, Try}
 
@@ -60,8 +62,9 @@ trait ElasticsearchComponent extends OaiRepository {
           .lte(until).format("strict_date_time")
 
 
-        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
-        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
+        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
+        val searchRequest = new SearchRequest().source(searchSourceBuilder)
+          .indices(index).scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
 
         Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
 
@@ -70,8 +73,9 @@ trait ElasticsearchComponent extends OaiRepository {
         val rqB = QueryBuilders
           .rangeQuery("lastUpdatedDate")
           .gte(from).format("strict_date_time")
-        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
-        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
+        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
+        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index)
+          .scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
 
         Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
 
@@ -80,8 +84,9 @@ trait ElasticsearchComponent extends OaiRepository {
         val rqB = QueryBuilders
           .rangeQuery("lastUpdatedDate")
           .lte(until).format("strict_date_time")
-        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
-        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
+        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
+        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index)
+          .scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
 
         Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
 
@@ -93,8 +98,9 @@ trait ElasticsearchComponent extends OaiRepository {
       //only for the moment until we have better data
       case (_, _,_,_,_) =>
         //for the moment
-        val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(30)
-        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
+        val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(listLength)
+        val searchRequest = new SearchRequest().source(searchSourceBuilder)
+          .indices(index).scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
 
         Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
 
@@ -108,16 +114,18 @@ trait ElasticsearchComponent extends OaiRepository {
         //condition to finish the fetching is to compare the length of the resultlist
         //compare: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html
         //otherwise we will get into an empty loop
-        val scrollId = if (searchResponse.getHits.getHits.length < 30) None else Option(searchResponse.getScrollId)
+        val scrollId = if (searchResponse.getHits.getHits.length < listLength) None else Option(searchResponse.getScrollId)
         val resumptionToken: Option[ResumptionToken] = scrollId.map(
           ESResumptionTokenHelper(_))
 
         val contentList = searchResponse.getHits.getHits.
           map(hit => {
             val source = hit.getSourceAsMap.asScala
+
             OAIContent(hit.getId,
               source.getOrElse("id","").toString,
-              source.getOrElse("document","").toString,
+              if (source.contains("document")) (Helper.decodeBase64FromString(source("document").toString)) else "",
+              //source.getOrElse("document","").toString,
               source.getOrElse("format","").toString,
               source.getOrElse("published",false).asInstanceOf[Boolean],
               source.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
@@ -161,7 +169,9 @@ trait ElasticsearchComponent extends OaiRepository {
       getResponse.getId
       OAIContent(getResponse.getId,
         hit.getOrElse("id", "").toString,
-        hit.getOrElse("document", "").toString,
+        //todo: test it
+        if (hit.contains("document")) (Helper.decodeBase64FromString(hit("document").toString)) else "",
+        //hit.getOrElse("document", "").toString,
         hit.getOrElse("format", "").toString,
         hit.getOrElse("published", false).asInstanceOf[Boolean],
         hit.getOrElse("recordset", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,

app/modules/OaiRepository.scala

@@ -52,6 +52,12 @@ trait OaiRepository {
                metadataPrefix: String
                ): Try[OAIContent]
 
+  def listLength: Int
+
+  def resumptionTokenTTL: Long
+
+
+
 
 
 }

app/org/swissbib/memobase/oai/common/validation/ParameterTypes.scala

@@ -109,6 +109,12 @@ case class NoContentForQueryParameter (verb: OaiVerb
                                      ,errorCode: String ) extends OaiParameterBase
 
 
+case class SystemErrorParameter (verb: OaiVerb
+                                       ,allQueryParameter: Map[String, Seq[String]]
+                                       ,errorCode: String
+                                        ,errorMessage: String) extends OaiParameterBase
+
+
 
 object OAIErrorCode extends Enumeration {
 
@@ -124,6 +130,7 @@ object OAIErrorCode extends Enumeration {
 
   val NO_RECORDS_MATCH: OAIErrorCodeVal = OAIErrorCodeVal("noRecordsMatch")
   val ID_DOES_NOT_EXIST: OAIErrorCodeVal = OAIErrorCodeVal("idDoesNotExist")
+  val SYSTEM_ERROR: OAIErrorCodeVal = OAIErrorCodeVal("idSystemError")
 
 
   def getErrorCode(code: Option[String]): Try[OAIErrorCode] = {

app/org/swissbib/memobase/oai/response/ListIdentifiersResponse.scala

@@ -58,7 +58,7 @@ case class ListIdentifiersResponse( config: Configuration
               )
               }
               {if (resultList.repositoryToken.isDefined) {
-              <resumptionToken>{resultList.repositoryToken}</resumptionToken>
+              <resumptionToken>{resultList.repositoryToken.get.token}</resumptionToken>
             }}
 
             </ListIdentifiers>

app/org/swissbib/memobase/oai/response/OaiResponse.scala

@@ -37,15 +37,9 @@ abstract class OaiResponse (oaiConfig: OaiConfig
                  datestamp: Instant = Instant.now(),
                  identifier: String): Elem =
     <header status={if (del) Some(Text("deleted")) else None}>
-      <identifier>
-        {identifier}
-      </identifier>
-      <datestamp>
-        {datestamp}
-      </datestamp>
-      <setSpec>
-        {set}
-      </setSpec>
+      <identifier>{identifier}</identifier>
+      <datestamp>{datestamp}</datestamp>
+      <setSpec>{set}</setSpec>
     </header>
 
 

app/org/swissbib/memobase/oai/response/SystemErrorResponse.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
+package org.swissbib.memobase.oai.response
+
+import com.typesafe.config.ConfigObject
+import modules.OaiRepository
+import org.swissbib.memobase.oai.common.validation.{BadArgumentsParameter, SystemErrorParameter}
+import play.api.Configuration
+
+import scala.collection.immutable
+import scala.xml.transform.{RewriteRule, RuleTransformer}
+import scala.xml.{Elem, Node}
+
+case class SystemErrorResponse(config: Configuration
+                               , repository: OaiRepository
+                               , parameter: SystemErrorParameter) extends OaiResponse (repository.oaiConfig
+                                                                                          ,parameter) {
+  override def toString: String = s"Ich bin eine BadArguments response "
+
+
+  override def createResponse: Seq[Node] = {
+
+    //todo create the body using the executed action in repository
+    val oaiFrame = createOaiFrame
+    val sets: Seq[ConfigObject] = repository.oaiConfig.sets.sets
+    //val configTuples: Seq[(ConfigValue, ConfigValue)] = sets.map(f => (f.get("spec"),f.get("name")))
+
+    val verbBody: RewriteRule = new RewriteRule {
+      override def transform(n: Node): Seq[Node] = n match {
+        case elem: Elem if elem.label == "request" =>
+          elem ++ {
+                <error code={parameter.errorCode}>{parameter.errorMessage}</error>
+          }
+
+        case n => n
+      }
+    }
+    new RuleTransformer(verbBody).transform(oaiFrame)
+
+  }
+}

app/org/swissbib/memobase/oai/runner/GetRecordRunner.scala

@@ -23,7 +23,7 @@
 package org.swissbib.memobase.oai.runner
 
 import modules.{OAIContent, OaiRepository}
-import org.swissbib.memobase.oai.common.validation.{CheckedGetRecordParameter, NoContentForQueryParameter, OAIErrorCode}
+import org.swissbib.memobase.oai.common.validation.{CheckedGetRecordParameter, NoContentForQueryParameter, OAIErrorCode, SystemErrorParameter}
 import org.swissbib.memobase.oai.response.{GetRecordResponse, OaiResponse}
 import play.api.Configuration
 
@@ -38,11 +38,16 @@ case class GetRecordRunner(parameter: CheckedGetRecordParameter) extends OaiRunn
         GetRecordResponse(config, repository, result,parameter)
       case Failure(exception) =>
         exception match {
-          case _: GetRecordFailure => NoContentForQueryRunner(NoContentForQueryParameter(parameter.verb
+          case e: GetRecordFailure =>
+
+            NoContentForQueryRunner(NoContentForQueryParameter(parameter.verb
             , parameter.allQueryParameter
             , OAIErrorCode.ID_DOES_NOT_EXIST)).run()
             //todo: make something useful for system errors
-          case _  => ???
+          case ee  =>
+            SystemErrorRunner(SystemErrorParameter(parameter.verb,parameter.allQueryParameter,
+              OAIErrorCode.SYSTEM_ERROR,exception.getMessage)).run()
+
         }
     }
   }

app/org/swissbib/memobase/oai/runner/ListIdentifiersRunner.scala

@@ -23,13 +23,12 @@
 package org.swissbib.memobase.oai.runner
 
 import modules.OaiRepository
-import org.swissbib.memobase.oai.common.validation.{CheckedListIdentifiersParameter, CheckedListIdentifiersParameterExclusive, NoContentForQueryParameter, OAIErrorCode}
+import org.swissbib.memobase.oai.common.validation.{CheckedListIdentifiersParameter, CheckedListIdentifiersParameterExclusive, NoContentForQueryParameter, OAIErrorCode, SystemErrorParameter}
 import org.swissbib.memobase.oai.response.{ListIdentifiersResponse, ListRecordsResponse, OaiResponse}
 import play.api.Configuration
 
 import scala.util.{Failure, Success}
 
-//CheckedListIdentifiersParameter
 case class ListIdentifiersRunner(parameter: CheckedListIdentifiersParameter) extends OaiRunner {
 
   override def run()(implicit config: Configuration, repository: OaiRepository): OaiResponse = {
@@ -46,15 +45,11 @@ case class ListIdentifiersRunner(parameter: CheckedListIdentifiersParameter) ext
           case Nil => NoContentForQueryRunner(NoContentForQueryParameter(parameter.verb
             , parameter.allQueryParameter
             ,OAIErrorCode.NO_RECORDS_MATCH)).run()
-          //at least one result
           case _ =>  ListIdentifiersResponse(config,repository,requestResult, parameter)
         }
-      case Failure(exception) => ???
-
-        //todo: make something useful
-
-      //ListRecordsResponse(config,repository,null, repository.oaiConfig,parameter)
-
+      case Failure(exception) =>
+        SystemErrorRunner(SystemErrorParameter(parameter.verb,parameter.allQueryParameter,
+          OAIErrorCode.SYSTEM_ERROR,exception.getMessage)).run()
     }
 
 

app/org/swissbib/memobase/oai/runner/ListRecordsRunner.scala

@@ -25,7 +25,7 @@
 package org.swissbib.memobase.oai.runner
 
 import modules.OaiRepository
-import org.swissbib.memobase.oai.common.validation.{CheckedListRecordsParameter, CheckedListRecordsParameterExclusive, NoContentForQueryParameter, OAIErrorCode}
+import org.swissbib.memobase.oai.common.validation.{CheckedListRecordsParameter, CheckedListRecordsParameterExclusive, NoContentForQueryParameter, OAIErrorCode, SystemErrorParameter}
 import org.swissbib.memobase.oai.response.{ListRecordsResponse, OaiResponse}
 import play.api.Configuration
 
@@ -51,8 +51,9 @@ case class ListRecordsRunner(parameter: CheckedListRecordsParameter) extends Oai
           //at least one result
           case _ =>  ListRecordsResponse(config,repository,requestResult, parameter)
         }
-      case Failure(exception) => ???
-      //ListRecordsResponse(config,repository,null, repository.oaiConfig,parameter)
+      case Failure(exception) =>
+        SystemErrorRunner(SystemErrorParameter(parameter.verb,parameter.allQueryParameter,
+          OAIErrorCode.SYSTEM_ERROR,exception.getMessage)).run()
     }
   }
 }

app/org/swissbib/memobase/oai/runner/OaiRunner.scala

@@ -23,8 +23,8 @@
 package org.swissbib.memobase.oai.runner
 
 import modules.OaiRepository
-import org.swissbib.memobase.oai.common.validation.{BadArgumentsParameter, NoContentForQueryParameter}
-import org.swissbib.memobase.oai.response.{BadArgumentsResponse, NoContentForQueryResponse, OaiResponse}
+import org.swissbib.memobase.oai.common.validation.{BadArgumentsParameter, NoContentForQueryParameter, SystemErrorParameter}
+import org.swissbib.memobase.oai.response.{BadArgumentsResponse, NoContentForQueryResponse, OaiResponse, SystemErrorResponse}
 import play.api.Configuration
 
 
@@ -56,4 +56,12 @@ case class BadArgumentsErrorRunner(parameter: BadArgumentsParameter) extends Oai
     BadArgumentsResponse(config, repository,parameter)
 }
 
+case class SystemErrorRunner(parameter: SystemErrorParameter) extends OaiRunner {
+
+  override def run()(implicit config: Configuration, repository: OaiRepository): OaiResponse =
+    SystemErrorResponse(config, repository,parameter)
+}
+
+
+
 

app/utilities/Helper.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
+package utilities
+
+import java.io.ByteArrayOutputStream
+import java.util.Base64
+import java.util.zip.Inflater
+
+object Helper {
+
+  def deserialize(data: Array[Byte]): String = {
+    val decompressor = new Inflater()
+    decompressor.setInput(data)
+    val bos = new ByteArrayOutputStream(data.length)
+    val buffer = new Array[Byte](8192)
+    while (!decompressor.finished) {
+
+      val size = decompressor.inflate(buffer)
+      bos.write(buffer, 0, size)
+    }
+    bos.toString
+
+  }
+
+  def decodeBase64FromString(encodedBase64String: String): String = {
+    deserialize(Base64.getDecoder.decode(encodedBase64String))
+  }
+
+}

build.sbt

@@ -19,6 +19,7 @@ lazy val root = (project in file("."))
     assemblyJarName in assembly := "app.jar",
     //mainClass in assembly := Some("ch.memobase.rico2edm.Main"),
     test in assembly := {},
+    /*
     assemblyMergeStrategy in assembly := {
       case "log4j.properties" => MergeStrategy.first
       case "module-info.class" => MergeStrategy.discard
@@ -36,7 +37,7 @@ lazy val root = (project in file("."))
         oldStrategy(x)
 
     },
-
+*/
 
     //assemblyJarName in assembly := "app.jar",
     //test in assembly := {},

conf/application.conf

@@ -19,21 +19,6 @@ repository.dispatcher {
 }
 
 
-elasticsearch {
-
-  //template_prefix="production"
-  //hosts: ["sb-ues5.swissbib.unibas.ch:8080","sb-ues6.swissbib.unibas.ch:8080","sb-ues7.swissbib.unibas.ch:8080","sb-ues8.swissbib.unibas.ch:8080"]
-  hosts: ["mb-es1.memobase.unibas.ch:8080"]
-  //hosts: ["localhost:8080"]
-  cluster: "test-memobase-search-cluster" # "weywot"
-  index: "oai-v1"
-  content: "application/json; charset=utf-8"
-  #the default number of result per page
-  defaultsize: 10
-  #the maximum number of result per page you can request (without scroll)
-  maxsize: 1000
-
-}
 
 kafka : [
   {

k8s-manifests/oai-prod-deploy.yaml

@@ -20,10 +20,23 @@ spec:
 
       containers:
         - name: oai-api-prod-container
-          image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/externalapis/oai:develop
-          #image: image: guenterh/oaimemobase:v0.5
+          #image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/externalapis/oai:latest
+          image: guenterh/oaimemobase:0.5.3
           ports:
             - containerPort: 9000
               name: http
               protocol: TCP
           imagePullPolicy: Always
+          env:
+            - name: RESPONSE_LISTLENGTH
+              value: "30"
+            - name: RESUMPTION_TOKEN_TTL
+              value: "3"
+            - name: ELASTICSEARCH_INDEX
+              value: oai-v2
+            - name: ELASTICSEARCH_HOSTS
+              value: localhost
+            - name: ELASTICSEARCH_PORT
+              value: "8080"
+            - name: ELASTICSEARCH_CLUSTER
+              value: test-memobase-search-cluster
\ No newline at end of file

local/env.txt

+RESPONSE_LISTLENGTH=30
+RESUMPTION_TOKEN_TTL=3
+ELASTICSEARCH_INDEX=oai-v2
+ELASTICSEARCH_HOSTS=localhost
+ELASTICSEARCH_PORT=8080
+ELASTICSEARCH_CLUSTER=test-memobase-search-cluster

project/Dependencies.scala

@@ -23,7 +23,7 @@ object Dependencies {
 
 
   lazy val joda = "org.joda" % "joda-convert" % "2.2.1"
-  lazy val logging = "net.logstash.logback" % "logstash-logback-encoder" % "6.6"
+  lazy val logging = "net.logstash.logback" % "logstash-logback-encoder" % "6.2"
   lazy val lemonlabs = "io.lemonlabs" %% "scala-uri" % "1.5.1"
   lazy val codingwell = "net.codingwell" %% "scala-guice" % "4.2.6"
   //"org.apache.solr" % "solr-solrj" % "7.3.1",