foundation for the next step where requests throwing a failure situation are...

foundation for the next step where requests throwing a failure situation are handled by dedicated error response handler
(a lot of files weren't included in the former commit)

app/ErrorHandler.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 import javax.inject.{Inject, Provider}
 
 import play.api._

app/RequestHandler.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 import javax.inject.Inject
 import play.api.OptionalDevContext
 import play.api.http._

app/controllers/BindController.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
+
 package controllers
 
 import controllers.BindController.AgeRange

app/controllers/OaiController.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 package controllers
 
 import javax.inject.Inject

app/modules/ElasticsearchClient.scala

-package modules
+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
 
-import java.io.IOException
-import java.util
 
-import com.typesafe.config.{Config, ConfigObject}
+package modules
+
+import com.typesafe.config.Config
 import javax.inject.{Inject, Singleton}
 import org.apache.http.{Header, HttpHost}
 import org.apache.http.message.BasicHeader
-import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest
-import org.elasticsearch.client.{RequestOptions, RestClient, RestHighLevelClient}
-import org.elasticsearch.common.bytes.BytesReference
-import org.elasticsearch.common.xcontent.{XContentFactory, XContentType}
+import org.elasticsearch.client.{RestClient, RestHighLevelClient}
 import org.swissbib.memobase.oai.common.util.{OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig}
 import play.Environment
 import play.api.inject.ApplicationLifecycle
-import utilities.FileUtil
 import scala.jdk.CollectionConverters._
-
 import scala.collection.mutable.ArrayBuffer
 import scala.concurrent.Future
 

app/modules/ElasticsearchComponent.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 package modules
 
 import java.util
@@ -6,20 +28,13 @@ import org.elasticsearch.action.get.GetRequest
 import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
 import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
 import org.elasticsearch.common.unit.TimeValue
-import org.elasticsearch.index.query.{QueryBuilders, RangeQueryBuilder}
+import org.elasticsearch.index.query.QueryBuilders
 import org.elasticsearch.search.builder.SearchSourceBuilder
-import org.joda.time.DateTime
 import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
-import org.swissbib.memobase.oai.runner.{ResultList, ResultListNew}
+import org.swissbib.memobase.oai.runner.{GetRecordFailure, ResultList}
 import play.Environment
-import play.api.libs.json.Json
-import utilities.Json2XML
-
-import java.time.format.DateTimeFormatter
-import scala.collection.mutable
 import scala.jdk.CollectionConverters._
 import scala.util.{Failure, Success, Try}
-import scala.xml.Elem
 
 trait ElasticsearchComponent extends OaiRepository {
   val client: Option[RestHighLevelClient]
@@ -33,7 +48,7 @@ trait ElasticsearchComponent extends OaiRepository {
                            until: Option[String],
                            set: Option[String],
                            resumptionToken: Option[ResumptionToken],
-                           metadataPrefix: String): Option[ResultList] = {
+                           metadataPrefix: String): Try[ResultList] = Try{
     //todo - more / better / tested handling for different parameters
 
     val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
@@ -55,7 +70,7 @@ trait ElasticsearchComponent extends OaiRepository {
         val rqB = QueryBuilders
           .rangeQuery("lastUpdatedDate")
           .gte(from).format("strict_date_time")
-        val searchSourceBuilder = new SearchSourceBuilder().query(rqB)
+        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
         val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
 
         Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
@@ -65,7 +80,7 @@ trait ElasticsearchComponent extends OaiRepository {
         val rqB = QueryBuilders
           .rangeQuery("lastUpdatedDate")
           .lte(until).format("strict_date_time")
-        val searchSourceBuilder = new SearchSourceBuilder().query(rqB)
+        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
         val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
 
         Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
@@ -78,7 +93,7 @@ trait ElasticsearchComponent extends OaiRepository {
       //only for the moment until we have better data
       case (_, _,_,_,_) =>
         //for the moment
-        val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
+        val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(30)
         val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
 
         Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
@@ -88,7 +103,6 @@ trait ElasticsearchComponent extends OaiRepository {
 
     searchResponse match  {
       case Success(searchResponse) =>
-        val si = searchResponse.getScrollId
 
         //scroll_id is always the same -> for this specific context
         //condition to finish the fetching is to compare the length of the resultlist
@@ -114,10 +128,8 @@ trait ElasticsearchComponent extends OaiRepository {
           ).toSeq
 
 
-        Option( ResultList(resumptionToken, Option(contentList)))
-      case Failure(exception) =>
-        //todo we have to do something!!
-        Option.empty
+        ResultList(resumptionToken, contentList)
+      case Failure(exception) => throw exception
     }
 
   }
@@ -128,7 +140,7 @@ trait ElasticsearchComponent extends OaiRepository {
                              until: Option[String],
                              set: Option[String],
                              resumptionToken: Option[ResumptionToken],
-                             metadataPrefix: String): Option[ResultList] =
+                             metadataPrefix: String): Try[ResultList] =
     listRecords(from,
                 until,
                 set,
@@ -137,39 +149,28 @@ trait ElasticsearchComponent extends OaiRepository {
 
 
   override def getRecord(identifier: String,
-                         metadataPrefix: String): Option[(String, OAIContent)] = {
+                         metadataPrefix: String): Try[OAIContent] = Try {
 
     val getRequest = new GetRequest(index, identifier)
     val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT)
 
-    //todo: what if response is empty?
-    val sM: Option[(String, util.Map[String, AnyRef])] = Option( (getResponse.getId,getResponse.getSourceAsMap))
-    //sM.map(tuple => (tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
-    sM.map(tuple => {
-
-     val esId = tuple._1
-     val esSourceMap = tuple._2.asScala
-
-      (
-        esId,
-      OAIContent(esId,
-        esSourceMap.getOrElse("id","").toString,
-        esSourceMap.getOrElse("document","").toString,
-        esSourceMap.getOrElse("format","").toString,
-        esSourceMap.getOrElse("published",false).asInstanceOf[Boolean],
-        esSourceMap.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
-        esSourceMap.getOrElse("institution",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
-        esSourceMap.getOrElse("lastUpdatedDate", "").asInstanceOf[String])
+    if (getResponse.getSource.isEmpty) {
+      throw GetRecordFailure("record was not found")
+    } else {
+      val hit = getResponse.getSource.asScala
+      getResponse.getId
+      OAIContent(getResponse.getId,
+        hit.getOrElse("id", "").toString,
+        hit.getOrElse("document", "").toString,
+        hit.getOrElse("format", "").toString,
+        hit.getOrElse("published", false).asInstanceOf[Boolean],
+        hit.getOrElse("recordset", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
+        hit.getOrElse("institution", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
+        hit.getOrElse("lastUpdatedDate", "").asInstanceOf[String]
       )
 
-      //(tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
 
-    //val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
-    //  sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
-    //    Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
-    //Option(myMap.toSeq)
+    }
 
-    })
   }
-
 }

app/modules/ElasticsearchModule.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 package modules
 
   import com.google.inject.AbstractModule
@@ -5,11 +27,6 @@ package modules
 class ElasticsearchModule extends AbstractModule{
 
   override def configure(): Unit = {
-    /*
-    bind(classOf[ElasticsearchComponent])
-      .to(classOf[ElasticsearchClient])
-      .asEagerSingleton()
-    */
     bind(classOf[OaiRepository])
       .to(classOf[ElasticsearchClient])
       .asEagerSingleton()

app/modules/KafkaComponent.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 package modules
 
 import com.typesafe.config.Config
@@ -14,8 +36,5 @@ class KafkaComponentImpl @Inject()(lifecycle: ApplicationLifecycle,
                                    config: Config,
                                    environment: Environment) extends KafkaComponent {
 
-  val t1: Config = config
-  val t2: Environment = environment
-  println("in kafka component")
 
 }

app/modules/KafkaModule.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 package modules
 
 import com.google.inject.AbstractModule

app/modules/OaiRepository.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 package modules
 
 import org.swissbib.memobase.oai.common.util.{OaiConfig, ResumptionToken}
-import org.swissbib.memobase.oai.runner.{ResultList, ResultListNew}
+import org.swissbib.memobase.oai.runner.ResultList
+
+import scala.util.Try
 
-import scala.xml.{Elem, Node}
 
 
 case class OAIContent(esId: String, docId: String, document: String, format: String, published: Boolean,set: List[String], institution: List[String], updateDate: String )
@@ -15,7 +38,7 @@ trait OaiRepository {
                   until: Option[String],
                  set: Option[String],
                  resumptionToken: Option[ResumptionToken],
-                 metadataPrefix: String): Option[ResultList]
+                 metadataPrefix: String): Try[ResultList]
 
 
 
@@ -23,11 +46,11 @@ trait OaiRepository {
                   until: Option[String],
                   set: Option[String],
                   resumptionToken: Option[ResumptionToken],
-                  metadataPrefix: String): Option[ResultList]
+                  metadataPrefix: String): Try[ResultList]
 
   def getRecord(identifier: String,
                metadataPrefix: String
-               ): Option[(String, OAIContent)]
+               ): Try[OAIContent]
 
 
 

app/org/swissbib/memobase/oai/common/util/OaiConfig.scala

+/*
+ * generic OAI Server - agnostic in relation to the data repository
+ * initially created for memobase project
+ *
+ * Copyright (C) 2021  UB Basel
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU Affero General Public License as
+ * published by the Free Software Foundation, either version 3 of the
+ * License, or (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU Affero General Public License for more details.
+ *
+ * You should have received a copy of the GNU Affero General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+
 package org.swissbib.memobase.oai.common.util