implementation of from / until parameters for List* verbs

app/modules/ElasticsearchComponent.scala

@@ -6,7 +6,7 @@ import org.elasticsearch.action.get.GetRequest
 import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
 import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
 import org.elasticsearch.common.unit.TimeValue
-import org.elasticsearch.index.query.QueryBuilders
+import org.elasticsearch.index.query.{QueryBuilders, RangeQueryBuilder}
 import org.elasticsearch.search.builder.SearchSourceBuilder
 import org.joda.time.DateTime
 import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
@@ -36,83 +36,52 @@ trait ElasticsearchComponent extends OaiRepository {
                            metadataPrefix: String): Option[ResultList] = {
     //todo - more / better / tested handling for different parameters
 
-    //scroll API: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/master/java-rest-high-search-scroll.html
-
     val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
       case (Some(from), Some(until),_,None,_) =>
-        //for the moment
-        val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
-        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
-
-        Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
-
-      case (_, _,_,Some(resumptionToken),_) =>
-        val scrollRequest = new SearchScrollRequest(resumptionToken.subject);
-        scrollRequest.scroll(TimeValue.timeValueMinutes(3L));
-        Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT);}
-        //only for the moment until we have better data
-      case (_, _,_,_,_) =>
-      //for the moment
-        val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
-        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
-
-        Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
-
-    }
-
-
-    searchResponse match  {
-      case Success(searchResponse) =>
-        val si = searchResponse.getScrollId
-
-        //scroll_id is always the same -> for this specific context
-        //condition to finish the fetching is to compare the length of the resultlist
-        //compare: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html
-        //otherwise we will get into an empty loop
-        val scrollId = if (searchResponse.getHits.getHits.length == 0) Option.empty[String] else Option(searchResponse.getScrollId)
-        val resumptionToken: Option[ResumptionToken] = scrollId.map(
-          ESResumptionTokenHelper(_))
 
+        val rqB = QueryBuilders
+          .rangeQuery("lastUpdatedDate")
+          .gte(from).format("strict_date_time")
+          .lte(until).format("strict_date_time")
 
 
-        val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
-          sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
-            Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
-
+        val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
+        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
 
-        Option( ResultList(resumptionToken, Option(myMap.toSeq)))
-      case Failure(exception) =>
-        //todo we have to do something!!
-        Option.empty
-    }
+        Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
 
-  }
+      case (Some(from), None ,_,None,_) =>
+        //for the moment
+        val rqB = QueryBuilders
+          .rangeQuery("lastUpdatedDate")
+          .gte(from).format("strict_date_time")
+        val searchSourceBuilder = new SearchSourceBuilder().query(rqB)
+        val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
 
-  override def listRecords1(from: Option[String],
-                            until: Option[String],
-                            set: Option[String],
-                            resumptionToken: Option[ResumptionToken],
-                            metadataPrefix: String): Option[ResultListNew] = {
+        Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
 
-    val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
-      case (Some(from), Some(until),_,None,_) =>
+      case (None, Some(until) ,_,None,_) =>
         //for the moment
-        val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
+        val rqB = QueryBuilders
+          .rangeQuery("lastUpdatedDate")
+          .lte(until).format("strict_date_time")
+        val searchSourceBuilder = new SearchSourceBuilder().query(rqB)
         val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
 
-        Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
+        Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
+
 
       case (_, _,_,Some(resumptionToken),_) =>
-        val scrollRequest = new SearchScrollRequest(resumptionToken.subject);
-        scrollRequest.scroll(TimeValue.timeValueMinutes(3L));
-        Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT);}
+        val scrollRequest = new SearchScrollRequest(resumptionToken.subject)
+        scrollRequest.scroll(TimeValue.timeValueMinutes(3L))
+        Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT)}
       //only for the moment until we have better data
       case (_, _,_,_,_) =>
         //for the moment
         val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
         val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
 
-        Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
+        Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
 
     }
 
@@ -125,7 +94,7 @@ trait ElasticsearchComponent extends OaiRepository {
         //condition to finish the fetching is to compare the length of the resultlist
         //compare: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html
         //otherwise we will get into an empty loop
-        val scrollId = if (searchResponse.getHits.getHits.length == 0) Option.empty[String] else Option(searchResponse.getScrollId)
+        val scrollId = if (searchResponse.getHits.getHits.length < 30) None else Option(searchResponse.getScrollId)
         val resumptionToken: Option[ResumptionToken] = scrollId.map(
           ESResumptionTokenHelper(_))
 
@@ -145,7 +114,7 @@ trait ElasticsearchComponent extends OaiRepository {
           ).toSeq
 
 
-        Option( ResultListNew(resumptionToken, Option(contentList)))
+        Option( ResultList(resumptionToken, Option(contentList)))
       case Failure(exception) =>
         //todo we have to do something!!
         Option.empty
@@ -154,6 +123,7 @@ trait ElasticsearchComponent extends OaiRepository {
   }
 
 
+
   override def listIdentiers(from: Option[String],
                              until: Option[String],
                              set: Option[String],
@@ -167,21 +137,39 @@ trait ElasticsearchComponent extends OaiRepository {
 
 
   override def getRecord(identifier: String,
-                         metadataPrefix: String): Option[(String, Elem)] = {
+                         metadataPrefix: String): Option[(String, OAIContent)] = {
 
     val getRequest = new GetRequest(index, identifier)
     val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT)
 
     //todo: what if response is empty?
     val sM: Option[(String, util.Map[String, AnyRef])] = Option( (getResponse.getId,getResponse.getSourceAsMap))
-    sM.map(tuple => (tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
+    //sM.map(tuple => (tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
+    sM.map(tuple => {
+
+     val esId = tuple._1
+     val esSourceMap = tuple._2.asScala
+
+      (
+        esId,
+      OAIContent(esId,
+        esSourceMap.getOrElse("id","").toString,
+        esSourceMap.getOrElse("document","").toString,
+        esSourceMap.getOrElse("format","").toString,
+        esSourceMap.getOrElse("published",false).asInstanceOf[Boolean],
+        esSourceMap.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
+        esSourceMap.getOrElse("institution",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
+        esSourceMap.getOrElse("lastUpdatedDate", "").asInstanceOf[String])
+      )
+
+      //(tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
 
     //val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
     //  sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
     //    Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
     //Option(myMap.toSeq)
 
-
+    })
   }
 
 }

app/modules/OaiRepository.scala

@@ -17,12 +17,6 @@ trait OaiRepository {
                  resumptionToken: Option[ResumptionToken],
                  metadataPrefix: String): Option[ResultList]
 
-  def listRecords1(from: Option[String],
-                  until: Option[String],
-                  set: Option[String],
-                  resumptionToken: Option[ResumptionToken],
-                  metadataPrefix: String): Option[ResultListNew]
-
 
 
   def listIdentiers(from: Option[String],
@@ -33,7 +27,7 @@ trait OaiRepository {
 
   def getRecord(identifier: String,
                metadataPrefix: String
-               ): Option[(String, Elem)]
+               ): Option[(String, OAIContent)]
 
 
 

app/org/swissbib/memobase/oai/response/OaiResponse.scala

@@ -80,7 +80,8 @@ case class GetRecordResponse(runner: GetRecordRunner) extends OaiResponse {
                 makeHeader(identifier = runner.result.get._1) }
               {if (runner.result.nonEmpty)
                 <metadata>
-                  {runner.result.get._2}
+
+                  {XML.loadString(runner.result.get._2.document)}
                 </metadata>}
                 {if (runner.result.isEmpty)
                   <metadata/>}
@@ -154,7 +155,8 @@ case class ListIdentifiersResponse(runner: OaiRequestRunner) extends OaiResponse
               {
               runner.resultList.get.result.get.map(
                 node => {
-                    {makeHeader(identifier = node._1)}
+                    {makeHeader(identifier = node.docId,
+                    datestamp = Instant.parse(node.updateDate))}
                 }
               )
               }
@@ -225,10 +227,11 @@ case class ListRecordsResponse(runner: OaiRequestRunner) extends OaiResponse {
           elem ++
           <ListRecords>
             {
-            runner.resultListnew.get.result.get.map(
+            runner.resultList.get.result.get.map(
               node => {
                 <record>
-                {makeHeader(identifier = node.docId)}
+                {makeHeader(identifier = node.docId,
+                datestamp = Instant.parse(node.updateDate))}
                 <metadata>{XML.loadString(node.document)}</metadata>
                 </record>
               }
@@ -243,8 +246,8 @@ case class ListRecordsResponse(runner: OaiRequestRunner) extends OaiResponse {
             )
 
              */
-            {if (runner.resultListnew.get.repositoryToken.isDefined) {
-              <resumptionToken>{runner.resultListnew.get.repositoryToken.get.token}</resumptionToken>
+            {if (runner.resultList.get.repositoryToken.isDefined) {
+              <resumptionToken>{runner.resultList.get.repositoryToken.get.token}</resumptionToken>
             }}
             }
 

app/org/swissbib/memobase/oai/runner/OaiRequestRunner.scala

 package org.swissbib.memobase.oai.runner
 
-import modules.OaiRepository
+import modules.{OAIContent, OaiRepository}
 import org.swissbib.memobase.oai.request.{BadArgumentsReq, GetRecordReq, IdentifyReq, ListIdentifiersReq, ListIdentifiersReqExclusive, ListMetadataFormatsReq, ListRecordsReq, ListRecordsReqExclusive, ListSetsReq, ListSetsReqExclusive}
 import org.swissbib.memobase.oai.response.{BadArgumentsResponse, GetRecordResponse, IdentifyResponse, ListIdentifiersResponse, ListMetadaFormatsResponse, ListRecordsResponse, ListSetsResponse, OaiResponse}
 import play.api.Configuration
 
-import scala.xml.{Elem, Node}
 
 sealed abstract class OaiRequestRunner {
-  var result: Option[(String, Elem)] = None
+  var result: Option[(String, OAIContent)] = None
   var resultList: Option[ResultList] = None
-  var resultListnew: Option[ResultListNew] = None
   //var result: Option[NodeSeq] = None
   def run(): OaiResponse
   val config: Configuration
@@ -77,21 +75,13 @@ case class ListRecordsRunner(config: Configuration, repository: OaiRepository, r
                                                                             ListRecordsReq) extends OaiRequestRunner {
   override def run(): OaiResponse = {
     //ich benötige einen genrischen Typ result
-    /*
+
     resultList = repository.listRecords(
                                     from = request.parameter.from,
                                     until = request.parameter.until,
                                     set = request.parameter.set,
                                     resumptionToken = Option.empty,
                                     metadataPrefix = request.parameter.metadataPrefix)
-    */
-
-    resultListnew = repository.listRecords1(
-      from = request.parameter.from,
-      until = request.parameter.until,
-      set = request.parameter.set,
-      resumptionToken = Option.empty,
-      metadataPrefix = request.parameter.metadataPrefix)
 
 
 

app/org/swissbib/memobase/oai/runner/ResultData.scala

@@ -5,7 +5,7 @@ import org.swissbib.memobase.oai.common.util.ResumptionToken
 
 import scala.xml.Node
 
-case class ResultList(repositoryToken:Option[ResumptionToken], result:Option[Seq[(String,Node)]])
+case class ResultList(repositoryToken:Option[ResumptionToken], result:Option[Seq[OAIContent]])
 
 case class ResultListNew(repositoryToken:Option[ResumptionToken], result:Option[Seq[OAIContent]])
 

notes/beispiele.dnb.txt

+1) GetRecord
+
+https://services.dnb.de/oai/repository?verb=GetRecord&metadataPrefix=MARC21-xml&identifier=oai:dnb.de/authorities/118540238
+
+wenn der record nicht vorhanden ist
+
+<?xml version="1.0" encoding="UTF-8"?>
+<OAI-PMH
+    xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
+    xmlns="http://www.openarchives.org/OAI/2.0/"
+    xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
+    <responseDate>2021-03-16T10:56:05Z</responseDate>
+    <request metadataPrefix="MARC21-xml" verb="GetRecord"
+        identifier="oai:dnb.de/authorities/11854023"
+        >https://services.dnb.de/oai/repository</request>
+    <error code="idDoesNotExist"/>
+</OAI-PMH>
+
+- welche error codes verwende ich?
+- wie implementiere ich das?
+
+

notes/memobase.oai.aufrufe.txt

@@ -11,7 +11,7 @@ mit anschliessendem resumption token (max. 2 Minuten Zeitdauer zwischen den Aufr
 https://oai.memobase.k8s.unibas.ch/?verb=ListRecords&resumptionToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzM4NCJ9.eyJpc3MiOiJtZW1vYmFzZSBPQUkgc2VydmVyIiwic3ViIjoiRG5GMVpYSjVWR2hsYmtabGRHTm9Bd0FBQUFBQUFzZjBGbFZLZERSeFJXZEdVazVYVmpSdGJsWXpNWGQ1VjBFQUFBQUFBQTRyTkJadVdWWXllV0ZCV2xORFF6aHNRM1JCZDBkRFoxQkJBQUFBQUFBT0t6VVdibGxXTW5saFFWcFRRME00YkVOMFFYZEhRMmRRUVE9PSIsImV4cCI6MTYwMTI5Mzk0OCwiaWF0IjoxNjAxMjkzNzY4fQ.CmuujiNhY8zzRtRnlpOsgplbyNpPqWjtVtf-dIvD6PLsJuhIwS5ETHGQW_KK4mcS
 
 ListIdentifiers
-https://oai.memobase.k8s.unibas.ch/?verb=ListIdentifiers&metadataPrefi  x=1234
+https://oai.memobase.k8s.unibas.ch/?verb=ListIdentifiers&metadataPrefix=1234
 mit anschliessendem resumptionToken - maximal 2 Minuten Zeitdauer zwischen den Aufrufen
 aktuell Fehler!
 https://oai.memobase.k8s.unibas.ch/?verb=ListIdentifiers&resumptionToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzM4NCJ9.eyJpc3MiOiJtZW1vYmFzZSBPQUkgc2VydmVyIiwic3ViIjoiRG5GMVpYSjVWR2hsYmtabGRHTm9Bd0FBQUFBQUFFeWFGbHAyZVVwb2FXWnpVVVJUTUdSdGVUbDZiSE4wY2tFQUFBQUFBQTRyZVJadVdWWXllV0ZCV2xORFF6aHNRM1JCZDBkRFoxQkJBQUFBQUFBT0szb1dibGxXTW5saFFWcFRRME00YkVOMFFYZEhRMmRRUVE9PSIsImV4cCI6MTYwMTI5NDE2NywiaWF0IjoxNjAxMjkzOTg3fQ.6dKO-1VRY4gybSBiC-wpUNd8SHFqiBAkv1YqG5w9AJ-hQz6AnN4UkHLwRKX1tjN4