Commit 7166db9f authored by Günter Hipler's avatar Günter Hipler
Browse files

foundation for the next step where requests throwing a failure situation are...

foundation for the next step where requests throwing a failure situation are handled by dedicated error response handler
(a lot of files weren't included in the former commit)
parent 71fd85aa
Pipeline #23315 passed with stage
in 2 minutes and 15 seconds
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
import javax.inject.{Inject, Provider}
import play.api._
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
import javax.inject.Inject
import play.api.OptionalDevContext
import play.api.http._
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package controllers
import controllers.BindController.AgeRange
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package controllers
import javax.inject.Inject
......
package modules
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
import java.io.IOException
import java.util
import com.typesafe.config.{Config, ConfigObject}
package modules
import com.typesafe.config.Config
import javax.inject.{Inject, Singleton}
import org.apache.http.{Header, HttpHost}
import org.apache.http.message.BasicHeader
import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest
import org.elasticsearch.client.{RequestOptions, RestClient, RestHighLevelClient}
import org.elasticsearch.common.bytes.BytesReference
import org.elasticsearch.common.xcontent.{XContentFactory, XContentType}
import org.elasticsearch.client.{RestClient, RestHighLevelClient}
import org.swissbib.memobase.oai.common.util.{OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig}
import play.Environment
import play.api.inject.ApplicationLifecycle
import utilities.FileUtil
import scala.jdk.CollectionConverters._
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.Future
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import java.util
......@@ -6,20 +28,13 @@ import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.{QueryBuilders, RangeQueryBuilder}
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.joda.time.DateTime
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
import org.swissbib.memobase.oai.runner.{ResultList, ResultListNew}
import org.swissbib.memobase.oai.runner.{GetRecordFailure, ResultList}
import play.Environment
import play.api.libs.json.Json
import utilities.Json2XML
import java.time.format.DateTimeFormatter
import scala.collection.mutable
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}
import scala.xml.Elem
trait ElasticsearchComponent extends OaiRepository {
val client: Option[RestHighLevelClient]
......@@ -33,7 +48,7 @@ trait ElasticsearchComponent extends OaiRepository {
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList] = {
metadataPrefix: String): Try[ResultList] = Try{
//todo - more / better / tested handling for different parameters
val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
......@@ -55,7 +70,7 @@ trait ElasticsearchComponent extends OaiRepository {
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB)
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
......@@ -65,7 +80,7 @@ trait ElasticsearchComponent extends OaiRepository {
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB)
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
......@@ -78,7 +93,7 @@ trait ElasticsearchComponent extends OaiRepository {
//only for the moment until we have better data
case (_, _,_,_,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery())
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
......@@ -88,7 +103,6 @@ trait ElasticsearchComponent extends OaiRepository {
searchResponse match {
case Success(searchResponse) =>
val si = searchResponse.getScrollId
//scroll_id is always the same -> for this specific context
//condition to finish the fetching is to compare the length of the resultlist
......@@ -114,10 +128,8 @@ trait ElasticsearchComponent extends OaiRepository {
).toSeq
Option( ResultList(resumptionToken, Option(contentList)))
case Failure(exception) =>
//todo we have to do something!!
Option.empty
ResultList(resumptionToken, contentList)
case Failure(exception) => throw exception
}
}
......@@ -128,7 +140,7 @@ trait ElasticsearchComponent extends OaiRepository {
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList] =
metadataPrefix: String): Try[ResultList] =
listRecords(from,
until,
set,
......@@ -137,39 +149,28 @@ trait ElasticsearchComponent extends OaiRepository {
override def getRecord(identifier: String,
metadataPrefix: String): Option[(String, OAIContent)] = {
metadataPrefix: String): Try[OAIContent] = Try {
val getRequest = new GetRequest(index, identifier)
val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT)
//todo: what if response is empty?
val sM: Option[(String, util.Map[String, AnyRef])] = Option( (getResponse.getId,getResponse.getSourceAsMap))
//sM.map(tuple => (tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
sM.map(tuple => {
val esId = tuple._1
val esSourceMap = tuple._2.asScala
(
esId,
OAIContent(esId,
esSourceMap.getOrElse("id","").toString,
esSourceMap.getOrElse("document","").toString,
esSourceMap.getOrElse("format","").toString,
esSourceMap.getOrElse("published",false).asInstanceOf[Boolean],
esSourceMap.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
esSourceMap.getOrElse("institution",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
esSourceMap.getOrElse("lastUpdatedDate", "").asInstanceOf[String])
if (getResponse.getSource.isEmpty) {
throw GetRecordFailure("record was not found")
} else {
val hit = getResponse.getSource.asScala
getResponse.getId
OAIContent(getResponse.getId,
hit.getOrElse("id", "").toString,
hit.getOrElse("document", "").toString,
hit.getOrElse("format", "").toString,
hit.getOrElse("published", false).asInstanceOf[Boolean],
hit.getOrElse("recordset", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
hit.getOrElse("institution", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
hit.getOrElse("lastUpdatedDate", "").asInstanceOf[String]
)
//(tuple._1,Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(tuple._2).toString))))
//val myMap = searchResponse.getHits.getHits.map(hit => (hit.getId, hit.getSourceAsMap)).map(
// sourceTuple => (sourceTuple._1, Json2XML.singleJsonDoc2Xml(
// Json.parse(play.libs.Json.toJson(sourceTuple._2).toString))))
//Option(myMap.toSeq)
})
}
}
}
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import com.google.inject.AbstractModule
......@@ -5,11 +27,6 @@ package modules
class ElasticsearchModule extends AbstractModule{
override def configure(): Unit = {
/*
bind(classOf[ElasticsearchComponent])
.to(classOf[ElasticsearchClient])
.asEagerSingleton()
*/
bind(classOf[OaiRepository])
.to(classOf[ElasticsearchClient])
.asEagerSingleton()
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import com.typesafe.config.Config
......@@ -14,8 +36,5 @@ class KafkaComponentImpl @Inject()(lifecycle: ApplicationLifecycle,
config: Config,
environment: Environment) extends KafkaComponent {
val t1: Config = config
val t2: Environment = environment
println("in kafka component")
}
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import com.google.inject.AbstractModule
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import org.swissbib.memobase.oai.common.util.{OaiConfig, ResumptionToken}
import org.swissbib.memobase.oai.runner.{ResultList, ResultListNew}
import org.swissbib.memobase.oai.runner.ResultList
import scala.util.Try
import scala.xml.{Elem, Node}
case class OAIContent(esId: String, docId: String, document: String, format: String, published: Boolean,set: List[String], institution: List[String], updateDate: String )
......@@ -15,7 +38,7 @@ trait OaiRepository {
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList]
metadataPrefix: String): Try[ResultList]
......@@ -23,11 +46,11 @@ trait OaiRepository {
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Option[ResultList]
metadataPrefix: String): Try[ResultList]
def getRecord(identifier: String,
metadataPrefix: String
): Option[(String, OAIContent)]
): Try[OAIContent]
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package org.swissbib.memobase.oai.common.util
import com.typesafe.config.ConfigObject
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package org.swissbib.memobase.oai.common.util
import java.time.{Clock, ZoneId}
import pdi.jwt.{Jwt, JwtAlgorithm, JwtClaim, JwtHeader}
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.validation.ListRecordsValidation.checkParameterByName
import org.swissbib.memobase.oai.request.{BadArgumentsReq, BaseUserRequest, GetRecordReq, ListRecordsReq, ListRecordsReqExclusive}
import org.swissbib.memobase.oai.request.{BadArgumentsReq, BaseUserRequest, GetRecordReq}
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.request.{BadArgumentsReq, BaseUserRequest, IdentifyReq}
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,