Commit 7166db9f authored by Günter Hipler's avatar Günter Hipler
Browse files

foundation for the next step where requests throwing a failure situation are...

foundation for the next step where requests throwing a failure situation are handled by dedicated error response handler
(a lot of files weren't included in the former commit)
parent 71fd85aa
Pipeline #23315 passed with stage
in 2 minutes and 15 seconds
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
import javax.inject.{Inject, Provider} import javax.inject.{Inject, Provider}
import play.api._ import play.api._
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
import javax.inject.Inject import javax.inject.Inject
import play.api.OptionalDevContext import play.api.OptionalDevContext
import play.api.http._ import play.api.http._
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package controllers package controllers
import controllers.BindController.AgeRange import controllers.BindController.AgeRange
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package controllers package controllers
import javax.inject.Inject import javax.inject.Inject
......
package modules /*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
import java.io.IOException
import java.util
import com.typesafe.config.{Config, ConfigObject} package modules
import com.typesafe.config.Config
import javax.inject.{Inject, Singleton} import javax.inject.{Inject, Singleton}
import org.apache.http.{Header, HttpHost} import org.apache.http.{Header, HttpHost}
import org.apache.http.message.BasicHeader import org.apache.http.message.BasicHeader
import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest import org.elasticsearch.client.{RestClient, RestHighLevelClient}
import org.elasticsearch.client.{RequestOptions, RestClient, RestHighLevelClient}
import org.elasticsearch.common.bytes.BytesReference
import org.elasticsearch.common.xcontent.{XContentFactory, XContentType}
import org.swissbib.memobase.oai.common.util.{OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig} import org.swissbib.memobase.oai.common.util.{OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig}
import play.Environment import play.Environment
import play.api.inject.ApplicationLifecycle import play.api.inject.ApplicationLifecycle
import utilities.FileUtil
import scala.jdk.CollectionConverters._ import scala.jdk.CollectionConverters._
import scala.collection.mutable.ArrayBuffer import scala.collection.mutable.ArrayBuffer
import scala.concurrent.Future import scala.concurrent.Future
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules package modules
import java.util import java.util
...@@ -6,20 +28,13 @@ import org.elasticsearch.action.get.GetRequest ...@@ -6,20 +28,13 @@ import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest} import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
import org.elasticsearch.client.{RequestOptions, RestHighLevelClient} import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
import org.elasticsearch.common.unit.TimeValue import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.{QueryBuilders, RangeQueryBuilder} import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.builder.SearchSourceBuilder import org.elasticsearch.search.builder.SearchSourceBuilder
import org.joda.time.DateTime
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken} import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
import org.swissbib.memobase.oai.runner.{ResultList, ResultListNew} import org.swissbib.memobase.oai.runner.{GetRecordFailure, ResultList}
import play.Environment import play.Environment
import play.api.libs.json.Json
import utilities.Json2XML
import java.time.format.DateTimeFormatter
import scala.collection.mutable
import scala.jdk.CollectionConverters._ import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try} import scala.util.{Failure, Success, Try}
import scala.xml.Elem
trait ElasticsearchComponent extends OaiRepository { trait ElasticsearchComponent extends OaiRepository {
val client: Option[RestHighLevelClient] val client: Option[RestHighLevelClient]
...@@ -33,7 +48,7 @@ trait ElasticsearchComponent extends OaiRepository { ...@@ -33,7 +48,7 @@ trait ElasticsearchComponent extends OaiRepository {
until: Option[String], until: Option[String],
set: Option[String], set: Option[String],
resumptionToken: Option[ResumptionToken], resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList] = { metadataPrefix: String): Try[ResultList] = Try{
//todo - more / better / tested handling for different parameters //todo - more / better / tested handling for different parameters
val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match { val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
...@@ -55,7 +70,7 @@ trait ElasticsearchComponent extends OaiRepository { ...@@ -55,7 +70,7 @@ trait ElasticsearchComponent extends OaiRepository {
val rqB = QueryBuilders val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate") .rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time") .gte(from).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB) val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L)) val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)} Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
...@@ -65,7 +80,7 @@ trait ElasticsearchComponent extends OaiRepository { ...@@ -65,7 +80,7 @@ trait ElasticsearchComponent extends OaiRepository {
val rqB = QueryBuilders val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate") .rangeQuery("lastUpdatedDate")
.lte(until).format("strict_date_time") .lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB) val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L)) val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)} Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
...@@ -78,7 +93,7 @@ trait ElasticsearchComponent extends OaiRepository { ...@@ -78,7 +93,7 @@ trait ElasticsearchComponent extends OaiRepository {
//only for the moment until we have better data //only for the moment until we have better data
case (_, _,_,_,_) => case (_, _,_,_,_) =>
//for the moment //for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()) val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L)) val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)} Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
...@@ -88,7 +103,6 @@ trait ElasticsearchComponent extends OaiRepository { ...@@ -88,7 +103,6 @@ trait ElasticsearchComponent extends OaiRepository {
searchResponse match { searchResponse match {
case Success(searchResponse) => case Success(searchResponse) =>
val si = searchResponse.getScrollId
//scroll_id is always the same -> for this specific context //scroll_id is always the same -> for this specific context
//condition to finish the fetching is to compare the length of the resultlist //condition to finish the fetching is to compare the length of the resultlist
...@@ -114,10 +128,8 @@ trait ElasticsearchComponent extends OaiRepository { ...@@ -114,10 +128,8 @@ trait ElasticsearchComponent extends OaiRepository {
).toSeq ).toSeq
Option( ResultList(resumptionToken, Option(contentList))) ResultList(resumptionToken, contentList)
case Failure(exception) => case Failure(exception) => throw exception
//todo we have to do something!!
Option.empty
} }
} }
...@@ -128,7 +140,7 @@ trait ElasticsearchComponent extends OaiRepository { ...@@ -128,7 +140,7 @@ trait ElasticsearchComponent extends OaiRepository {
until: Option[String], until: Option[String],
set: Option[String], set: Option[String],
resumptionToken: Option[ResumptionToken], resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList] = metadataPrefix: String): Try[ResultList] =
listRecords(from, listRecords(from,
until, until,
set, set,
...@@ -137,39 +149,28 @@ trait ElasticsearchComponent extends OaiRepository { ...@@ -137,39 +149,28 @@ trait ElasticsearchComponent extends OaiRepository {
override def getRecord(identifier: String, override def getRecord(identifier: String,
metadataPrefix: String): Option[(String, OAIContent)] = { metadataPrefix: String): Try[OAIContent] = Try {
val getRequest = new GetRequest(index, identifier) val getRequest = new GetRequest(index, identifier)
val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT) val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT)
//todo: what if response is empty? if (getResponse.getSource.isEmpty) {
val sM: Option[(String, util.Map[String, AnyRef])] = Option( (getResponse.getId,getResponse.getSourceAsMap)) throw GetRecordFailure("record was not found")
//sM.map(tuple => (tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString)))) } else {
sM.map(tuple => { val hit = getResponse.getSource.asScala
getResponse.getId
val esId = tuple._1 OAIContent(getResponse.getId,
val esSourceMap = tuple._2.asScala hit.getOrElse("id", "").toString,
hit.getOrElse("document", "").toString,
( hit.getOrElse("format", "").toString,
esId, hit.getOrElse("published", false).asInstanceOf[Boolean],
OAIContent(esId, hit.getOrElse("recordset", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
esSourceMap.getOrElse("id","").toString, hit.getOrElse("institution", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
esSourceMap.getOrElse("document","").toString, hit.getOrElse("lastUpdatedDate", "").asInstanceOf[String]
esSourceMap.getOrElse("format","").toString,
esSourceMap.getOrElse("published",false).asInstanceOf[Boolean],
esSourceMap.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
esSourceMap.getOrElse("institution",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
esSourceMap.getOrElse("lastUpdatedDate", "").asInstanceOf[String])
) )
//(tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
//val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map( }
// sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
// Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
//Option(myMap.toSeq)
})
} }
} }
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules package modules
import com.google.inject.AbstractModule import com.google.inject.AbstractModule
...@@ -5,11 +27,6 @@ package modules ...@@ -5,11 +27,6 @@ package modules
class ElasticsearchModule extends AbstractModule{ class ElasticsearchModule extends AbstractModule{
override def configure(): Unit = { override def configure(): Unit = {
/*
bind(classOf[ElasticsearchComponent])
.to(classOf[ElasticsearchClient])
.asEagerSingleton()
*/
bind(classOf[OaiRepository]) bind(classOf[OaiRepository])
.to(classOf[ElasticsearchClient]) .to(classOf[ElasticsearchClient])
.asEagerSingleton() .asEagerSingleton()
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules package modules
import com.typesafe.config.Config import com.typesafe.config.Config
...@@ -14,8 +36,5 @@ class KafkaComponentImpl @Inject()(lifecycle: ApplicationLifecycle, ...@@ -14,8 +36,5 @@ class KafkaComponentImpl @Inject()(lifecycle: ApplicationLifecycle,
config: Config, config: Config,
environment: Environment) extends KafkaComponent { environment: Environment) extends KafkaComponent {
val t1: Config = config
val t2: Environment = environment
println("in kafka component")
} }
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules package modules
import com.google.inject.AbstractModule import com.google.inject.AbstractModule
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules package modules
import org.swissbib.memobase.oai.common.util.{OaiConfig, ResumptionToken} import org.swissbib.memobase.oai.common.util.{OaiConfig, ResumptionToken}
import org.swissbib.memobase.oai.runner.{ResultList, ResultListNew} import org.swissbib.memobase.oai.runner.ResultList
import scala.util.Try
import scala.xml.{Elem, Node}
case class OAIContent(esId: String, docId: String, document: String, format: String, published: Boolean,set: List[String], institution: List[String], updateDate: String ) case class OAIContent(esId: String, docId: String, document: String, format: String, published: Boolean,set: List[String], institution: List[String], updateDate: String )
...@@ -15,7 +38,7 @@ trait OaiRepository { ...@@ -15,7 +38,7 @@ trait OaiRepository {
until: Option[String], until: Option[String],
set: Option[String], set: Option[String],
resumptionToken: Option[ResumptionToken], resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList] metadataPrefix: String): Try[ResultList]
...@@ -23,11 +46,11 @@ trait OaiRepository { ...@@ -23,11 +46,11 @@ trait OaiRepository {
until: Option[String], until: Option[String],
set: Option[String], set: Option[String],
resumptionToken: Option[ResumptionToken], resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList] metadataPrefix: String): Try[ResultList]
def getRecord(identifier: String, def getRecord(identifier: String,
metadataPrefix: String metadataPrefix: String
): Option[(String, OAIContent)] ): Try[OAIContent]
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package org.swissbib.memobase.oai.common.util package org.swissbib.memobase.oai.common.util