Skip to content

GitLab

Commit 9cbdf261 authored by Günter Hipler's avatar Günter Hipler
Browse files

implementation of from / until parameters for List* verbs

parent 65edcd8a
Expand all Hide whitespace changes
Inline Side-by-side
app/modules/ElasticsearchComponent.scala
View file @ 9cbdf261
......@@ -6,7 +6,7 @@ import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.query.{QueryBuilders, RangeQueryBuilder}
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.joda.time.DateTime
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
......@@ -36,83 +36,52 @@ trait ElasticsearchComponent extends OaiRepository {
metadataPrefix: String): Option[ResultList] = {
//todo - more / better / tested handling for different parameters
//scroll API: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/master/java-rest-high-search-scroll.html
val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
case (Some(from), Some(until),_,None,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (_, _,_,Some(resumptionToken),_) =>
val scrollRequest = new SearchScrollRequest(resumptionToken.subject);
scrollRequest.scroll(TimeValue.timeValueMinutes(3L));
Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT);}
//only for the moment until we have better data
case (_, _,_,_,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
}
searchResponse match {
case Success(searchResponse) =>
val si = searchResponse.getScrollId
//scroll_id is always the same -> for this specific context
//condition to finish the fetching is to compare the length of the resultlist
//compare: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html
//otherwise we will get into an empty loop
val scrollId = if (searchResponse.getHits.getHits.length == 0) Option.empty[String] else Option(searchResponse.getScrollId)
val resumptionToken: Option[ResumptionToken] = scrollId.map(
ESResumptionTokenHelper(_))
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
.lte(until).format("strict_date_time")
val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Option( ResultList(resumptionToken, Option(myMap.toSeq)))
case Failure(exception) =>
//todo we have to do something!!
Option.empty
}
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
}
case (Some(from), None ,_,None,_) =>
//for the moment
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
override def listRecords1(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultListNew] = {
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
case (Some(from), Some(until),_,None,_) =>
case (None, Some(until) ,_,None,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (_, _,_,Some(resumptionToken),_) =>
val scrollRequest = new SearchScrollRequest(resumptionToken.subject);
scrollRequest.scroll(TimeValue.timeValueMinutes(3L));
Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT);}
val scrollRequest = new SearchScrollRequest(resumptionToken.subject)
scrollRequest.scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT)}
//only for the moment until we have better data
case (_, _,_,_,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
}
......@@ -125,7 +94,7 @@ trait ElasticsearchComponent extends OaiRepository {
//condition to finish the fetching is to compare the length of the resultlist
//compare: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html
//otherwise we will get into an empty loop
val scrollId = if (searchResponse.getHits.getHits.length == 0) Option.empty[String] else Option(searchResponse.getScrollId)
val scrollId = if (searchResponse.getHits.getHits.length < 30) None else Option(searchResponse.getScrollId)
val resumptionToken: Option[ResumptionToken] = scrollId.map(
ESResumptionTokenHelper(_))
......@@ -145,7 +114,7 @@ trait ElasticsearchComponent extends OaiRepository {
).toSeq
Option( ResultListNew(resumptionToken, Option(contentList)))
Option( ResultList(resumptionToken, Option(contentList)))
case Failure(exception) =>
//todo we have to do something!!
Option.empty
......@@ -154,6 +123,7 @@ trait ElasticsearchComponent extends OaiRepository {
}
override def listIdentiers(from: Option[String],
until: Option[String],
set: Option[String],
......@@ -167,21 +137,39 @@ trait ElasticsearchComponent extends OaiRepository {
override def getRecord(identifier: String,
metadataPrefix: String): Option[(String, Elem)] = {
metadataPrefix: String): Option[(String, OAIContent)] = {
val getRequest = new GetRequest(index, identifier)
val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT)
//todo: what if response is empty?
val sM: Option[(String, util.Map[String, AnyRef])] = Option( (getResponse.getId,getResponse.getSourceAsMap))
sM.map(tuple => (tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
//sM.map(tuple => (tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
sM.map(tuple => {
val esId = tuple._1
val esSourceMap = tuple._2.asScala
(
esId,
OAIContent(esId,
esSourceMap.getOrElse("id","").toString,
esSourceMap.getOrElse("document","").toString,
esSourceMap.getOrElse("format","").toString,
esSourceMap.getOrElse("published",false).asInstanceOf[Boolean],
esSourceMap.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
esSourceMap.getOrElse("institution",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
esSourceMap.getOrElse("lastUpdatedDate", "").asInstanceOf[String])
)
//(tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
//val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
// sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
// Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
//Option(myMap.toSeq)
})
}
}
app/modules/OaiRepository.scala
View file @ 9cbdf261
......@@ -17,12 +17,6 @@ trait OaiRepository {
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList]
def listRecords1(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultListNew]
def listIdentiers(from: Option[String],
......@@ -33,7 +27,7 @@ trait OaiRepository {
def getRecord(identifier: String,
metadataPrefix: String
): Option[(String, Elem)]
): Option[(String, OAIContent)]
......
app/org/swissbib/memobase/oai/response/OaiResponse.scala
View file @ 9cbdf261
......@@ -80,7 +80,8 @@ case class GetRecordResponse(runner: GetRecordRunner) extends OaiResponse {
makeHeader(identifier = runner.result.get._1) }
{if (runner.result.nonEmpty)
<metadata>
{runner.result.get._2}
{XML.loadString(runner.result.get._2.document)}
</metadata>}
{if (runner.result.isEmpty)
<metadata/>}
......@@ -154,7 +155,8 @@ case class ListIdentifiersResponse(runner: OaiRequestRunner) extends OaiResponse
{
runner.resultList.get.result.get.map(
node => {
{makeHeader(identifier = node._1)}
{makeHeader(identifier = node.docId,
datestamp = Instant.parse(node.updateDate))}
}
)
}
......@@ -225,10 +227,11 @@ case class ListRecordsResponse(runner: OaiRequestRunner) extends OaiResponse {
elem ++
<ListRecords>
{
runner.resultListnew.get.result.get.map(
runner.resultList.get.result.get.map(
node => {
<record>
{makeHeader(identifier = node.docId)}
{makeHeader(identifier = node.docId,
datestamp = Instant.parse(node.updateDate))}
<metadata>{XML.loadString(node.document)}</metadata>
</record>
}
......@@ -243,8 +246,8 @@ case class ListRecordsResponse(runner: OaiRequestRunner) extends OaiResponse {
)
*/
{if (runner.resultListnew.get.repositoryToken.isDefined) {
<resumptionToken>{runner.resultListnew.get.repositoryToken.get.token}</resumptionToken>
{if (runner.resultList.get.repositoryToken.isDefined) {
<resumptionToken>{runner.resultList.get.repositoryToken.get.token}</resumptionToken>
}}
}
......
app/org/swissbib/memobase/oai/runner/OaiRequestRunner.scala
View file @ 9cbdf261
package org.swissbib.memobase.oai.runner
import modules.OaiRepository
import modules.{OAIContent, OaiRepository}
import org.swissbib.memobase.oai.request.{BadArgumentsReq, GetRecordReq, IdentifyReq, ListIdentifiersReq, ListIdentifiersReqExclusive, ListMetadataFormatsReq, ListRecordsReq, ListRecordsReqExclusive, ListSetsReq, ListSetsReqExclusive}
import org.swissbib.memobase.oai.response.{BadArgumentsResponse, GetRecordResponse, IdentifyResponse, ListIdentifiersResponse, ListMetadaFormatsResponse, ListRecordsResponse, ListSetsResponse, OaiResponse}
import play.api.Configuration
import scala.xml.{Elem, Node}
sealed abstract class OaiRequestRunner {
var result: Option[(String, Elem)] = None
var result: Option[(String, OAIContent)] = None
var resultList: Option[ResultList] = None
var resultListnew: Option[ResultListNew] = None
//var result: Option[NodeSeq] = None
def run(): OaiResponse
val config: Configuration
......@@ -77,21 +75,13 @@ case class ListRecordsRunner(config: Configuration, repository: OaiRepository, r
ListRecordsReq) extends OaiRequestRunner {
override def run(): OaiResponse = {
//ich benötige einen genrischen Typ result
/*
resultList = repository.listRecords(
from = request.parameter.from,
until = request.parameter.until,
set = request.parameter.set,
resumptionToken = Option.empty,
metadataPrefix = request.parameter.metadataPrefix)
*/
resultListnew = repository.listRecords1(
from = request.parameter.from,
until = request.parameter.until,
set = request.parameter.set,
resumptionToken = Option.empty,
metadataPrefix = request.parameter.metadataPrefix)
......
app/org/swissbib/memobase/oai/runner/ResultData.scala
View file @ 9cbdf261
......@@ -5,7 +5,7 @@ import org.swissbib.memobase.oai.common.util.ResumptionToken
import scala.xml.Node
case class ResultList(repositoryToken:Option[ResumptionToken], result:Option[Seq[(String,Node)]])
case class ResultList(repositoryToken:Option[ResumptionToken], result:Option[Seq[OAIContent]])
case class ResultListNew(repositoryToken:Option[ResumptionToken], result:Option[Seq[OAIContent]])
......
notes/beispiele.dnb.txt 0 → 100644
View file @ 9cbdf261
1) GetRecord
https://services.dnb.de/oai/repository?verb=GetRecord&metadataPrefix=MARC21-xml&identifier=oai:dnb.de/authorities/118540238
wenn der record nicht vorhanden ist
<?xml version="1.0" encoding="UTF-8"?>
<OAI-PMH
xsi:schemaLocation="http://www.openarchives.org/OAI/2.0/ http://www.openarchives.org/OAI/2.0/OAI-PMH.xsd"
xmlns="http://www.openarchives.org/OAI/2.0/"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<responseDate>2021-03-16T10:56:05Z</responseDate>
<request metadataPrefix="MARC21-xml" verb="GetRecord"
identifier="oai:dnb.de/authorities/11854023"
>https://services.dnb.de/oai/repository</request>
<error code="idDoesNotExist"/>
</OAI-PMH>
- welche error codes verwende ich?
- wie implementiere ich das?
notes/memobase.oai.aufrufe.txt
View file @ 9cbdf261
......@@ -11,7 +11,7 @@ mit anschliessendem resumption token (max. 2 Minuten Zeitdauer zwischen den Aufr
https://oai.memobase.k8s.unibas.ch/?verb=ListRecords&resumptionToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzM4NCJ9.eyJpc3MiOiJtZW1vYmFzZSBPQUkgc2VydmVyIiwic3ViIjoiRG5GMVpYSjVWR2hsYmtabGRHTm9Bd0FBQUFBQUFzZjBGbFZLZERSeFJXZEdVazVYVmpSdGJsWXpNWGQ1VjBFQUFBQUFBQTRyTkJadVdWWXllV0ZCV2xORFF6aHNRM1JCZDBkRFoxQkJBQUFBQUFBT0t6VVdibGxXTW5saFFWcFRRME00YkVOMFFYZEhRMmRRUVE9PSIsImV4cCI6MTYwMTI5Mzk0OCwiaWF0IjoxNjAxMjkzNzY4fQ.CmuujiNhY8zzRtRnlpOsgplbyNpPqWjtVtf-dIvD6PLsJuhIwS5ETHGQW_KK4mcS
ListIdentifiers
https://oai.memobase.k8s.unibas.ch/?verb=ListIdentifiers&metadataPrefi x=1234
https://oai.memobase.k8s.unibas.ch/?verb=ListIdentifiers&metadataPrefix=1234
mit anschliessendem resumptionToken - maximal 2 Minuten Zeitdauer zwischen den Aufrufen
aktuell Fehler!
https://oai.memobase.k8s.unibas.ch/?verb=ListIdentifiers&resumptionToken=eyJ0eXAiOiJKV1QiLCJhbGciOiJIUzM4NCJ9.eyJpc3MiOiJtZW1vYmFzZSBPQUkgc2VydmVyIiwic3ViIjoiRG5GMVpYSjVWR2hsYmtabGRHTm9Bd0FBQUFBQUFFeWFGbHAyZVVwb2FXWnpVVVJUTUdSdGVUbDZiSE4wY2tFQUFBQUFBQTRyZVJadVdWWXllV0ZCV2xORFF6aHNRM1JCZDBkRFoxQkJBQUFBQUFBT0szb1dibGxXTW5saFFWcFRRME00YkVOMFFYZEhRMmRRUVE9PSIsImV4cCI6MTYwMTI5NDE2NywiaWF0IjoxNjAxMjkzOTg3fQ.6dKO-1VRY4gybSBiC-wpUNd8SHFqiBAkv1YqG5w9AJ-hQz6AnN4UkHLwRKX1tjN4
......
test.xml deleted 100644 → 0
View file @ 65edcd8a
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment