Commit 3e67055e authored by Günter Hipler's avatar Günter Hipler
Browse files

first resumption token requests possible

parent 24d9475b
Pipeline #14160 passed with stage
in 2 minutes and 25 seconds
......@@ -4,16 +4,19 @@ import java.util
import com.typesafe.config.Config
import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.search.{SearchRequest, SearchResponse}
import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.swissbib.memobase.oai.common.util.ResumptionToken
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
import org.swissbib.memobase.oai.runner.ResultList
import play.Environment
import play.api.libs.json.Json
import utilities.Json2XML
import scala.xml.{Elem, Node}
import scala.util.{Failure, Success, Try}
import scala.xml.Elem
trait ElasticsearchComponent extends OaiRepository {
val client: Option[RestHighLevelClient]
......@@ -27,24 +30,57 @@ trait ElasticsearchComponent extends OaiRepository {
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[Seq[(String,Node)]] = {
//todo - handling for different parameters
metadataPrefix: String): Option[ResultList] = {
//todo - more / better / tested handling for different parameters
//scroll API: https://www.elastic.co/guide/en/elasticsearch/client/java-rest/master/java-rest-high-search-scroll.html
val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
case (Some(from), Some(until),_,None,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (_, _,_,Some(resumptionToken),_) =>
val scrollRequest = new SearchScrollRequest(resumptionToken.subject);
scrollRequest.scroll(TimeValue.timeValueMinutes(3L));
Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT);}
//only for the moment until we have better data
case (_, _,_,_,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
}
searchResponse match {
case Success(searchResponse) =>
val si = searchResponse.getScrollId
println(si)
val resumptionToken: Option[ResumptionToken] = Option(searchResponse.getScrollId).map(
ESResumptionTokenHelper(_))
val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
Option( ResultList(resumptionToken, Option(myMap.toSeq)))
case Failure(exception) =>
//todo we have to do something!!
Option.empty
}
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index)
val searchResponse: SearchResponse = client.get.search(searchRequest, RequestOptions.DEFAULT)
val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
Option(myMap.toSeq)
}
override def listIdentiers(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[Seq[(String,Node)]] =
metadataPrefix: String): Option[ResultList] =
listRecords(from,
until,
set,
......@@ -69,4 +105,5 @@ trait ElasticsearchComponent extends OaiRepository {
}
}
package modules
import org.swissbib.memobase.oai.common.util.{OaiConfig, ResumptionToken}
import org.swissbib.memobase.oai.runner.ResultList
import scala.xml.{Elem, Node}
......@@ -10,13 +11,13 @@ trait OaiRepository {
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[Seq[(String,Node)]]
metadataPrefix: String): Option[ResultList]
def listIdentiers(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[Seq[(String,Node)]]
metadataPrefix: String): Option[ResultList]
def getRecord(identifier: String,
metadataPrefix: String
......
......@@ -9,18 +9,19 @@ import scala.util.{Failure, Success, Try}
case class ResumptionToken (issuer: String, issuedAt: Long, expiration: Long, subject: String, claim: JwtClaim)
case class ResumptionToken (issuer: String, issuedAt: Long, expiration: Long, subject: String, claim: JwtClaim, token:String)
object ESResumptionTokenHelper {
implicit val clock: Clock = Clock.system(ZoneId.of("Europe/Zurich"))
def apply(esCursor: String, expiresInSeconds: Long = 180): ResumptionToken = {
def apply(esCursor: String, expiresInSeconds: Long = 180, key:String = "oaimemobase"): ResumptionToken = {
val jwtClaim = JwtClaim(
subject = Option(esCursor),
issuer = Option("memobase OAI server")
).issuedNow.expiresIn(expiresInSeconds)
ResumptionToken( jwtClaim.issuer.get,jwtClaim.issuedAt.get,jwtClaim.expiration.get,jwtClaim.subject.get,jwtClaim)
val tokenWithClaim = Jwt.encode(JwtHeader(JwtAlgorithm.HS384), jwtClaim, key)
ResumptionToken( jwtClaim.issuer.get,jwtClaim.issuedAt.get,jwtClaim.expiration.get,jwtClaim.subject.get,jwtClaim,tokenWithClaim)
}
......@@ -28,18 +29,25 @@ object ESResumptionTokenHelper {
Jwt.encode(JwtHeader(JwtAlgorithm.HS384), rt.claim, key)
}
def createResumptionToken(oaiToken: String, key:String="oaimemobase"): Try[ResumptionToken] = {
def createResumptionTokenFromOaiToken(oaiToken: String, key:String="oaimemobase",expiresInSeconds: Long = 180): Try[ResumptionToken] = {
val decoded = Jwt.decodeRaw(oaiToken, key, Seq(JwtAlgorithm.HS384))
decoded match {
case Success(value) =>
val jsnonNode: JsValue = Json.parse(value)
//val issuer = (jsnonNode \ "iss").as[String]
//val subject = (jsnonNode \ "sub").as[String]
val issuer = (jsnonNode \ "iss").as[String]
val subject = (jsnonNode \ "sub").as[String]
val expiration = (jsnonNode \ "exp").as[Long]
val issuedAt = (jsnonNode \ "iat").as[Long]
Success(ResumptionToken("lll",expiration,issuedAt,"mySubject",JwtClaim("lll").issuedNow.expiresIn(10)))
val jwtClaim = JwtClaim(
subject = Option(subject),
issuer = Option("memobase OAI server")
).issuedAt(issuedAt).expiresIn(expiresInSeconds)
val tokenWithClaim = Jwt.encode(JwtHeader(JwtAlgorithm.HS384), jwtClaim, key)
Success(ResumptionToken(issuer,expiration,issuedAt,subject,jwtClaim,tokenWithClaim))
case Failure(value) => Failure(value)
}
......@@ -56,7 +64,7 @@ object Test extends App {
val token = ESResumptionTokenHelper.createOaiToken(rt)
//Thread.sleep(6000)
ESResumptionTokenHelper.createResumptionToken(token) match {
ESResumptionTokenHelper.createResumptionTokenFromOaiToken(token) match {
case Success(value) => println(value)
case Failure(value) => println(value)
}
......
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.util.ESResumptionTokenHelper
import org.swissbib.memobase.oai.common.validation.GetRecordValidation.{checkMissingParameterByName, checkParameterByName}
import org.swissbib.memobase.oai.common.validation.ListRecordsValidation.checkParameterByName
import org.swissbib.memobase.oai.request.{BadArgumentsReq, BaseUserRequest, ListIdentifiersReq, ListIdentifiersReqExclusive, ListRecordsReq, ListRecordsReqExclusive}
import scala.util.{Failure, Success}
/*
https://www.openarchives.org/OAI/openarchivesprotocol.html#ListIdentifiers
......@@ -42,11 +46,16 @@ object ListIdentifiersValidation extends ParameterValidationFunction {
//only resumption token with nothing else is possible
case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, Some(resumptionToken), allParameter, configuration) =>
//todo: check for valid resumption token
checkParameterByName(allParameter, AllowedParameter.listIdentifiersResumption) match {
case Seq() => ListIdentifiersReqExclusive(CheckedListIdentifiersParameterExclusive(verb.get, resumptionToken))
case setWithIllegalItems => BadArgumentsReq(BadArgumentsParameter(verb.get, setWithIllegalItems.map(key => (key, key)).toMap))
}
ESResumptionTokenHelper.createResumptionTokenFromOaiToken(resumptionToken) match {
case Success(rt) =>
checkParameterByName(allParameter, AllowedParameter.listIdentifiersResumption) match {
case Seq() => ListIdentifiersReqExclusive(CheckedListIdentifiersParameterExclusive(verb.get, rt))
case setWithIllegalItems => BadArgumentsReq(BadArgumentsParameter(verb.get, setWithIllegalItems.map(key => (key, key)).toMap))
}
case Failure(exceptionRT) =>
BadArgumentsReq(BadArgumentsParameter(verb.get, Map("resumptionToken" -> exceptionRT.getMessage)))
}
//todo: make it right - only for completion here
case OaiCheckedVerbWithParameter(verb, _, _, _, _, _, _, allParameter,_ ) =>
//todo: ich muss noch einen Weg finden herauszufinden, welche Kombination dieser Parameter dazu führt, dass diese
......
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.validation.GetRecordValidation.{checkMissingParameterByName, checkParameterByName}
import org.swissbib.memobase.oai.common.verb.OaiVerb
import org.swissbib.memobase.oai.common.util.ESResumptionTokenHelper
import org.swissbib.memobase.oai.request.{BadArgumentsReq, BaseUserRequest, IdentifyReq, ListRecordsReq, ListRecordsReqExclusive}
import scala.util.{Failure, Success}
......@@ -48,12 +49,18 @@ object ListRecordsValidation extends ParameterValidationFunction {
ListRecordsReq(CheckedListRecordsParameter(verb.get,mp, from, until, set))
//only resumption token with nothing else is possible
case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, resumptionToken, allParameter, configuration) =>
//todo: check for valid resumption token
checkParameterByName(allParameter, AllowedParameter.listRecordResumption) match {
case Seq() => ListRecordsReqExclusive(CheckedListRecordsParameterExclusive(verb.get, resumptionToken.get))
case setWithIllegalItems => BadArgumentsReq(BadArgumentsParameter(verb.get, setWithIllegalItems.map(key => (key, key)).toMap))
ESResumptionTokenHelper.createResumptionTokenFromOaiToken(resumptionToken.get) match {
case Success(rt) =>
checkParameterByName(allParameter, AllowedParameter.listRecordResumption) match {
case Seq() => ListRecordsReqExclusive(CheckedListRecordsParameterExclusive(verb.get, rt))
case setWithIllegalItems => BadArgumentsReq(BadArgumentsParameter(verb.get, setWithIllegalItems.map(key => (key, key)).toMap))
}
case Failure(exceptionRT) =>
BadArgumentsReq(BadArgumentsParameter(verb.get, Map("resumptionToken" -> exceptionRT.getMessage)))
}
case OaiCheckedVerbWithParameter(verb, _, _, _, _, _, _, allParameter, configuration) =>
//todo: das nochmals anschauen
......
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.verb.OaiVerb.{GetRecord, Identify, ListIdentifiers, ListMetadataFormats,
ListRecords, ListSets, OaiVerb}
import org.swissbib.memobase.oai.common.util.ResumptionToken
import org.swissbib.memobase.oai.common.verb.OaiVerb.{GetRecord, Identify, ListIdentifiers, ListMetadataFormats, ListRecords, ListSets, OaiVerb}
sealed abstract class OaiParameterBase() extends Product with Serializable {
def listRequestParameters: Map[String,String]
......@@ -38,7 +38,7 @@ case class CheckedListIdentifiersParameter (verb:OaiVerb, metadataPrefix: String
override def listRequestParameters: Map[String, String] = Map()
}
case class CheckedListIdentifiersParameterExclusive (verb:OaiVerb, resumptionToken: String) extends OaiParameterBase {
case class CheckedListIdentifiersParameterExclusive (verb:OaiVerb, resumptionToken: ResumptionToken) extends OaiParameterBase {
require(verb == ListIdentifiers)
override def listRequestParameters: Map[String, String] = Map()
}
......@@ -58,7 +58,7 @@ final case class CheckedListRecordsParameter (verb:OaiVerb,
override def listRequestParameters: Map[String, String] = Map()
}
final case class CheckedListRecordsParameterExclusive (verb:OaiVerb, resumptionToken: String) extends OaiParameterBase {
final case class CheckedListRecordsParameterExclusive (verb:OaiVerb, resumptionToken: ResumptionToken) extends OaiParameterBase {
require(verb == ListRecords)
override def listRequestParameters: Map[String, String] = Map()
}
......
package org.swissbib.memobase.oai.response
import java.time.Instant
import com.typesafe.config.{ConfigObject, ConfigValue}
import org.swissbib.memobase.oai.common.validation.OaiParameterBase
import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, GetRecordRunner, IdentifyRunner, OaiRequestRunner}
import scala.collection.immutable
......@@ -152,12 +150,16 @@ case class ListIdentifiersResponse(runner: OaiRequestRunner) extends OaiResponse
elem ++
<ListIdentifiers>
{
runner.resultList.get.map(
runner.resultList.get.result.get.map(
node => {
{makeHeader(identifier = node._1)}
}
)
}
{if (runner.resultList.get.repositoryToken.isDefined) {
<resumptionToken>{runner.resultList.get.repositoryToken.get.token}</resumptionToken>
}}
</ListIdentifiers>
case n => n
}
......@@ -221,7 +223,7 @@ case class ListRecordsResponse(runner: OaiRequestRunner) extends OaiResponse {
elem ++
<ListRecords>
{
runner.resultList.get.map(
runner.resultList.get.result.get.map(
node => {
<record>
{makeHeader(identifier = node._1)}
......@@ -230,6 +232,22 @@ case class ListRecordsResponse(runner: OaiRequestRunner) extends OaiResponse {
}
)
}
{
/*
why doesn't this work??
runner.resultList.get.repositoryToken.map(
tokenvalue =>
(<resumptionToken>{tokenvalue}</resumptionToken>).get
)
*/
{if (runner.resultList.get.repositoryToken.isDefined) {
<resumptionToken>{runner.resultList.get.repositoryToken.get.token}</resumptionToken>
}}
}
</ListRecords>
case n => n
}
......
......@@ -9,7 +9,7 @@ import scala.xml.{Elem, Node}
sealed abstract class OaiRequestRunner {
var result: Option[(String, Elem)] = None
var resultList: Option[Seq[(String,Node)]] = None
var resultList: Option[ResultList] = None
//var result: Option[NodeSeq] = None
def run(): OaiResponse
val config: Configuration
......@@ -96,7 +96,7 @@ case class ListRecordsExclusiveRunner(config: Configuration, repository: OaiRepo
from = Option.empty,
until = Option.empty,
set = Option.empty,
resumptionToken = Option.empty,
resumptionToken = Option(request.parameter.resumptionToken),
metadataPrefix = "4711")
......
package org.swissbib.memobase.oai.runner
import org.swissbib.memobase.oai.common.util.ResumptionToken
import scala.xml.Node
case class ResultList(repositoryToken:Option[String],result:Option[Seq[(String,Node)]])
case class ResultList(repositoryToken:Option[ResumptionToken], result:Option[Seq[(String,Node)]])
case class Result()
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment