Commit 65edcd8a authored by Günter Hipler's avatar Günter Hipler
Browse files

first step include edm data - verb ListRecords

parent 4d2acb41
......@@ -3,6 +3,7 @@ package controllers
import javax.inject.Inject
import modules.OaiRepository
import org.swissbib.memobase.oai.common.validation.OaiQueryParameterChecker
import org.swissbib.memobase.oai.request.BaseUserRequest
import play.api.Configuration
import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents, Rendering}
......@@ -26,7 +27,7 @@ class OaiController @Inject()(cc: ControllerComponents,
Action { implicit request =>
val qP: Map[String, Seq[String]] = request.queryString
val userRequest = new OaiQueryParameterChecker(
val userRequest: BaseUserRequest = new OaiQueryParameterChecker(
verb = verb,
metadataPrefix = metadataPrefix,
set = set,
......
package modules
import java.util
import com.typesafe.config.Config
import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
......@@ -9,12 +8,16 @@ import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.joda.time.DateTime
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
import org.swissbib.memobase.oai.runner.ResultList
import org.swissbib.memobase.oai.runner.{ResultList, ResultListNew}
import play.Environment
import play.api.libs.json.Json
import utilities.Json2XML
import java.time.format.DateTimeFormatter
import scala.collection.mutable
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}
import scala.xml.Elem
......@@ -70,9 +73,13 @@ trait ElasticsearchComponent extends OaiRepository {
val resumptionToken: Option[ResumptionToken] = scrollId.map(
ESResumptionTokenHelper(_))
val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
Option( ResultList(resumptionToken, Option(myMap.toSeq)))
case Failure(exception) =>
//todo we have to do something!!
......@@ -81,6 +88,72 @@ trait ElasticsearchComponent extends OaiRepository {
}
override def listRecords1(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultListNew] = {
val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
case (Some(from), Some(until),_,None,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (_, _,_,Some(resumptionToken),_) =>
val scrollRequest = new SearchScrollRequest(resumptionToken.subject);
scrollRequest.scroll(TimeValue.timeValueMinutes(3L));
Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT);}
//only for the moment until we have better data
case (_, _,_,_,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.search(searchRequest, RequestOptions.DEFAULT)}
}
searchResponse match {
case Success(searchResponse) =>
val si = searchResponse.getScrollId
//scroll_id is always the same -> for this specific context
//condition to finish the fetching is to compare the length of the resultlist
//compare: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html
//otherwise we will get into an empty loop
val scrollId = if (searchResponse.getHits.getHits.length == 0) Option.empty[String] else Option(searchResponse.getScrollId)
val resumptionToken: Option[ResumptionToken] = scrollId.map(
ESResumptionTokenHelper(_))
val contentList = searchResponse.getHits.getHits.
map(hit => {
val source = hit.getSourceAsMap.asScala
OAIContent(hit.getId,
source.getOrElse("id","").toString,
source.getOrElse("document","").toString,
source.getOrElse("format","").toString,
source.getOrElse("published",false).asInstanceOf[Boolean],
source.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
source.getOrElse("institution",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
source.getOrElse("lastUpdatedDate", "").asInstanceOf[String])
}
).toSeq
Option( ResultListNew(resumptionToken, Option(contentList)))
case Failure(exception) =>
//todo we have to do something!!
Option.empty
}
}
override def listIdentiers(from: Option[String],
until: Option[String],
set: Option[String],
......
package modules
import org.swissbib.memobase.oai.common.util.{OaiConfig, ResumptionToken}
import org.swissbib.memobase.oai.runner.ResultList
import org.swissbib.memobase.oai.runner.{ResultList, ResultListNew}
import scala.xml.{Elem, Node}
case class OAIContent(esId: String, docId: String, document: String, format: String, published: Boolean,set: List[String], institution: List[String], updateDate: String )
trait OaiRepository {
val oaiConfig: OaiConfig
def listRecords(from: Option[String],
......@@ -13,6 +17,14 @@ trait OaiRepository {
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList]
def listRecords1(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultListNew]
def listIdentiers(from: Option[String],
until: Option[String],
set: Option[String],
......
......@@ -5,7 +5,7 @@ import com.typesafe.config.{ConfigObject, ConfigValue}
import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, GetRecordRunner, IdentifyRunner, OaiRequestRunner}
import scala.collection.immutable
import scala.xml.{Elem, Node, Text}
import scala.xml.{Elem, Node, Text, XML}
import scala.xml.transform.{RewriteRule, RuleTransformer}
sealed abstract class OaiResponse {
......@@ -225,11 +225,11 @@ case class ListRecordsResponse(runner: OaiRequestRunner) extends OaiResponse {
elem ++
<ListRecords>
{
runner.resultList.get.result.get.map(
runner.resultListnew.get.result.get.map(
node => {
<record>
{makeHeader(identifier = node._1)}
<metadata>{node._2}</metadata>
{makeHeader(identifier = node.docId)}
<metadata>{XML.loadString(node.document)}</metadata>
</record>
}
)
......@@ -243,8 +243,8 @@ case class ListRecordsResponse(runner: OaiRequestRunner) extends OaiResponse {
)
*/
{if (runner.resultList.get.repositoryToken.isDefined) {
<resumptionToken>{runner.resultList.get.repositoryToken.get.token}</resumptionToken>
{if (runner.resultListnew.get.repositoryToken.isDefined) {
<resumptionToken>{runner.resultListnew.get.repositoryToken.get.token}</resumptionToken>
}}
}
......
......@@ -10,6 +10,7 @@ import scala.xml.{Elem, Node}
sealed abstract class OaiRequestRunner {
var result: Option[(String, Elem)] = None
var resultList: Option[ResultList] = None
var resultListnew: Option[ResultListNew] = None
//var result: Option[NodeSeq] = None
def run(): OaiResponse
val config: Configuration
......@@ -76,12 +77,22 @@ case class ListRecordsRunner(config: Configuration, repository: OaiRepository, r
ListRecordsReq) extends OaiRequestRunner {
override def run(): OaiResponse = {
//ich benötige einen genrischen Typ result
/*
resultList = repository.listRecords(
from = request.parameter.from,
until = request.parameter.until,
set = request.parameter.set,
resumptionToken = Option.empty,
metadataPrefix = request.parameter.metadataPrefix)
*/
resultListnew = repository.listRecords1(
from = request.parameter.from,
until = request.parameter.until,
set = request.parameter.set,
resumptionToken = Option.empty,
metadataPrefix = request.parameter.metadataPrefix)
ListRecordsResponse(this)
......
package org.swissbib.memobase.oai.runner
import modules.OAIContent
import org.swissbib.memobase.oai.common.util.ResumptionToken
import scala.xml.Node
case class ResultList(repositoryToken:Option[ResumptionToken], result:Option[Seq[(String,Node)]])
case class ResultListNew(repositoryToken:Option[ResumptionToken], result:Option[Seq[OAIContent]])
case class Result()
......@@ -16,7 +16,7 @@ lazy val root = (project in file("."))
.enablePlugins(GitVersioning, PlayService, PlayLayoutPlugin, Common)
.settings(
name := "oaiServiceMemobase",
//assemblyJarName in assembly := "app.jar",
assemblyJarName in assembly := "app.jar",
//test in assembly := {},
//mainClass in assembly := Some("Main"),
resolvers ++= Seq(
......
......@@ -23,10 +23,10 @@ elasticsearch {
//template_prefix="production"
//hosts: ["sb-ues5.swissbib.unibas.ch:8080","sb-ues6.swissbib.unibas.ch:8080","sb-ues7.swissbib.unibas.ch:8080","sb-ues8.swissbib.unibas.ch:8080"]
hosts: ["mb-es1.memobase.unibas.ch:8080"]
//hosts: ["localhost:8080"]
//hosts: ["mb-es1.memobase.unibas.ch:8080"]
hosts: ["localhost:8080"]
cluster: "test-memobase-search-cluster" # "weywot"
index: "documents-v9"
index: "oai-v1"
content: "application/json; charset=utf-8"
#the default number of result per page
defaultsize: 10
......
kafkacat -b mb-ka1.memobase.unibas.ch:9092 -t fedora-output-json-records -p 1 -o -10 -e > testrecords.json
https://gitlab.switch.ch/memoriav/memobase-2020/services/postprocessing/iiif-manifest-creator
This diff is collapsed.
This diff is collapsed.
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment