Commit b5d4aafb authored by Günter Hipler's avatar Günter Hipler
Browse files

more validations for all verbs and OAIRepository (ES) supports now the

restrictions of sets and metadataPrefixes
parent 8c609416
Pipeline #23955 passed with stage
in 4 minutes and 46 seconds
......@@ -84,9 +84,9 @@ class ElasticsearchClient @Inject()(
//collection conversions
//https://stackoverflow.com/questions/8301947/what-is-the-difference-between-javaconverters-and-javaconversions-in-scala
val oaiSets = config.getObjectList("oaiconfigs.sets").asScala.toSeq.map(
val oaiSets:Map[String,OAISet] = config.getObjectList("oaiconfigs.sets").asScala.toSeq.map(
o => {
OAISet(
o.get("spec").unwrapped().toString -> OAISet(
spec = o.get("spec").unwrapped().toString,
name = o.get("name").unwrapped().toString,
includedSets = Option(o.getOrDefault("includedsets",null))
......@@ -95,7 +95,7 @@ class ElasticsearchClient @Inject()(
fieldValue = Option(o.getOrDefault("value",null)).map(_.unwrapped().toString)
)
}
)
).toMap
val oaiConfigPrefixes = config.getObjectList("oaiconfigs.metadataPrefix").asScala.toSeq
val oaiPrefixes = config.getObjectList("oaiconfigs.metadataPrefix").asScala.toSeq.map(
......@@ -110,7 +110,7 @@ class ElasticsearchClient @Inject()(
OaiConfig(
commonConfig,
identifyConfig,
oaiSets.toList,
oaiSets,
oaiPrefixes.toList )
}
......
......@@ -28,9 +28,10 @@ import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.index.query.{QueryBuilder, QueryBuilders, TermQueryBuilder}
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
import org.swissbib.memobase.oai.common.util.es.QueryBuilderHelper
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, OaiConfig, ResumptionToken}
import org.swissbib.memobase.oai.runner.{GetRecordFailure, ResultList}
import play.Environment
import utilities.Helper
......@@ -38,6 +39,7 @@ import utilities.Helper
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}
//noinspection ScalaStyle
trait ElasticsearchComponent extends OaiRepository {
val client: Option[RestHighLevelClient]
......@@ -54,54 +56,55 @@ trait ElasticsearchComponent extends OaiRepository {
//todo - more / better / tested handling for different parameters
val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
case (Some(from), Some(until),_,None,_) =>
case (Some(from), Some(until),set,None,metaData) =>
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
.lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
val searchRequest = new SearchRequest().source(searchSourceBuilder)
.indices(index).scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
val queries = if (set.isDefined) {
createSetBoolQuery(set.get,oaiConfig).must(from_untilRangeQuery(from,until))
} else {
from_untilRangeQuery(from,until)
}
val searchRequest = createScrollSearchRequest(metaData,queries)
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (Some(from), None ,_,None,_) =>
//for the moment
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index)
.scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
case (Some(from), None ,set,None,metaDataP) =>
val queries = if (set.isDefined) {
createSetBoolQuery(set.get,oaiConfig).must(fromRangeQuery(from))
} else {
fromRangeQuery(from)
}
val searchRequest = createScrollSearchRequest(metaDataP,queries)
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (None, Some(until) ,_,None,_) =>
//for the moment
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index)
.scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
case (None, Some(until) ,set,None,metaDataP) =>
val queries = if (set.isDefined) {
createSetBoolQuery(set.get,oaiConfig).must(untilRangeQuery(until))
} else {
untilRangeQuery(until)
}
val searchRequest = createScrollSearchRequest(metaDataP,queries)
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (_, _,_,Some(resumptionToken),_) =>
//use only the resumption token
val scrollRequest = new SearchScrollRequest(resumptionToken.subject)
scrollRequest.scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT)}
//only for the moment until we have better data
case (_, _,_,_,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(listLength)
val searchRequest = new SearchRequest().source(searchSourceBuilder)
.indices(index).scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
case (_, _,set,_,metadataP) =>
val queries = if (set.isDefined) {
createSetBoolQuery(set.get,oaiConfig)
} else {
QueryBuilders.matchAllQuery()
}
val searchRequest = createScrollSearchRequest(metadataP,queries)
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
}
......@@ -162,6 +165,10 @@ trait ElasticsearchComponent extends OaiRepository {
val getRequest = new GetRequest(index, identifier)
val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT)
//todo: by now metadataPrefix is not being considered although it's mandatory in the OAI standard
// we provide any legal format for any document
// should be changed ones we differentiate in more detail
if (getResponse.isSourceEmpty) {
throw GetRecordFailure("record was not found")
} else {
......@@ -179,8 +186,53 @@ trait ElasticsearchComponent extends OaiRepository {
hit.getOrElse("lastUpdatedDate", "").asInstanceOf[String]
)
}
}
private def createSetBoolQuery(set: String, oaiConfig: OaiConfig) = {
val queries: util.List[TermQueryBuilder] = QueryBuilderHelper.setSpecMatchQuery(set,oaiConfig).asJava
val boolQueryBuilder = QueryBuilders.boolQuery()
//todo: if we avoid throwing an exception setSpecMatchQuery
// use foldLeft but test if list is empty. In this case create matchAll (or throw an exception??)
queries.forEach(q => boolQueryBuilder.should(q))
boolQueryBuilder
}
private def from_untilRangeQuery(from: String, until: String) =
QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
.lte(until).format("strict_date_time")
private def fromRangeQuery(from: String) =
QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
private def untilRangeQuery(until: String) =
QueryBuilders
.rangeQuery("lastUpdatedDate")
.lte(until).format("strict_date_time")
private def composeMetadataLeaf(metaDataPrefix: String, qb: QueryBuilder) =
QueryBuilders.boolQuery()
.must(qb)
.must(QueryBuilders.termQuery("format",metaDataPrefix))
private def createScrollSearchRequest(metaDataPrefix: String, queries: QueryBuilder) = {
val searchSourceBuilder = new SearchSourceBuilder()
.query(composeMetadataLeaf(metaDataPrefix,queries)).size(listLength)
//println(searchSourceBuilder.toString)
new SearchRequest().source(searchSourceBuilder)
.indices(index).scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
}
}
......@@ -26,7 +26,7 @@ import com.typesafe.config.ConfigObject
case class OaiConfig(common: OaiCommonConfig,
identify: OaiIdentifyConfig,
sets: List[OAISet],
sets: Map[String,OAISet],
prefixes: List[OAIMetadataPrefix])
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package org.swissbib.memobase.oai.common.util.es
import org.swissbib.memobase.oai.common.util.OaiConfig
import org.elasticsearch.index.query.TermQueryBuilder
object QueryBuilderHelper {
//@tailrec todo: make it tail recursive
def setSpecMatchQuery(set: String, oaiConfig: OaiConfig): List[TermQueryBuilder] = {
val setInQuery = oaiConfig.sets(set)
if (setInQuery.field.isDefined && setInQuery.fieldValue.isDefined) {
List(new TermQueryBuilder(setInQuery.field.get,setInQuery.fieldValue.get))
} else if (setInQuery.includedSets.isDefined) {
setInQuery.includedSets.get.foldLeft(
List[List[TermQueryBuilder]]())((list, set) =>
setSpecMatchQuery(set, oaiConfig) :: list).flatten
} else {
//todo: maybe an empty list would be more adequate
throw new Exception("set definiton not consistent")
}
}
}
......@@ -23,6 +23,7 @@
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.validation.ListRecordsValidation.checkValidMetadaPrefix
import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, GetRecordRunner, OaiRunner}
......@@ -45,13 +46,20 @@ object GetRecordValidation extends ParameterValidationFunction {
v1 match {
//possible valid combination
case OaiCheckedVerbWithParameter(verb, Some(metadataPrefix), None, None, None, Some(ident), None, allParameter, configuration) =>
checkParameterByName(allParameter, AllowedParameter.getRecord) match {
case Seq() => GetRecordRunner(CheckedGetRecordParameter(verb.get, ident, metadataPrefix, allParameter ))
case setWithIllegalItems =>
BadArgumentsErrorRunner(BadArgumentsParameter(verb.get
,setWithIllegalItems.map(key => (key, key)).toMap
,allParameter))
case OaiCheckedVerbWithParameter(Some(verb), Some(metadataPrefix), None, None, None, Some(ident), None, allParameter, configuration) =>
if (!checkValidMetadaPrefix(metadataPrefix,configuration)) {
BadArgumentsErrorRunner(BadArgumentsParameter(verb,
Map("metadataPrefix" -> s"$metadataPrefix no valid specification"),
allParameter))
} else {
checkParameterByName(allParameter, AllowedParameter.getRecord) match {
case Seq() => GetRecordRunner(CheckedGetRecordParameter(verb, ident, metadataPrefix, allParameter))
case setWithIllegalItems =>
BadArgumentsErrorRunner(BadArgumentsParameter(verb
, setWithIllegalItems.map(key => (key, key)).toMap
, allParameter))
}
}
//todo: make it right - only for completion here
......
......@@ -23,7 +23,8 @@
package org.swissbib.memobase.oai.common.validation
import org.swissbib.memobase.oai.common.util.ESResumptionTokenHelper
import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, ListIdentifiersExclusiveRunner, ListIdentifiersRunner, OaiRunner}
import org.swissbib.memobase.oai.common.validation.ListRecordsValidation.{checkParameterByName, checkValidMetadaPrefix, checkValidSet}
import org.swissbib.memobase.oai.runner.{BadArgumentsErrorRunner, ListIdentifiersExclusiveRunner, ListIdentifiersRunner, ListRecordsExclusiveRunner, ListRecordsRunner, OaiRunner}
import scala.util.{Failure, Success}
......@@ -47,51 +48,104 @@ noSetHierarchy - The repository does not support sets.
*/
//noinspection ScalaStyle
object ListIdentifiersValidation extends ParameterValidationFunction {
override def apply(v1: OaiCheckedVerbWithParameter): OaiRunner =
v1 match {
//possible valid combination
case OaiCheckedVerbWithParameter(verb, Some(mp), set, from, until, None, None, allParameter, configuration) =>
case OaiCheckedVerbWithParameter(Some(verb), Some(metadataPrefix), Some(set), from, until, None, None, allParameter, configuration) =>
//todo: check valid metadataPrefix
// check for additional illegal parameter
// check until not before from
// check until not before from - really ???
// check for valid dates
checkParameterByName(allParameter, AllowedParameter.listIdentifiers) match {
case Seq() => ListIdentifiersRunner(CheckedListIdentifiersParameter(verb.get, mp, set, from, until, allParameter))
case setWithIllegalItems => BadArgumentsErrorRunner(BadArgumentsParameter(verb.get
,setWithIllegalItems.map(key => (key, key)).toMap
,allParameter))
checkParameterByName(allParameter, AllowedParameter.listIdentifiersAllAllowed) match {
case Seq() =>
if (!checkValidSet(set,configuration)) {
BadArgumentsErrorRunner(BadArgumentsParameter(verb,
Map("setSpec" -> s"$set no valid specification"),
allParameter))
}
else if (!checkValidMetadaPrefix(metadataPrefix,configuration)) {
BadArgumentsErrorRunner(BadArgumentsParameter(verb,
Map("metadataPrefix" -> s"$metadataPrefix no valid specification"),
allParameter))
} else {
ListIdentifiersRunner(CheckedListIdentifiersParameter(verb, metadataPrefix, Some(set), from, until, allParameter))
}
case setWithIllegalItems =>
BadArgumentsErrorRunner(BadArgumentsParameter(verb, setWithIllegalItems.map(key => (key, "")).toMap
, allParameter))
}
//only resumption token with nothing else is possible
//todo: hier falle ich hinein wenn ich "kein" metadata prefix habe aber auch kein resumption token, dann gibt es
// einen Fehler!
// dringend beheben!
case OaiCheckedVerbWithParameter(Some(verb), Some(metadataPrefix),None, from, until, None, None, allParameter
//request with no set
, configuration) =>
checkParameterByName(allParameter, AllowedParameter.listIdentifiersAllAllowed) match {
case Seq() =>
if (!checkValidMetadaPrefix(metadataPrefix,configuration)) {
BadArgumentsErrorRunner(BadArgumentsParameter(verb,
Map("metadataPrefix" -> s"$metadataPrefix no valid specification"),
allParameter))
} else {
ListIdentifiersRunner(CheckedListIdentifiersParameter(verb, metadataPrefix, None, from, until, allParameter))
}
case setWithIllegalItems =>
BadArgumentsErrorRunner(BadArgumentsParameter(verb, setWithIllegalItems.map(key => (key, "")).toMap
, allParameter))
}
//only resumption token with nothing else is possible
case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, Some(resumptionToken), allParameter, configuration) =>
//todo: check for valid resumption token
ESResumptionTokenHelper.createResumptionTokenFromOaiToken(resumptionToken) match {
case Success(rt) =>
checkParameterByName(allParameter, AllowedParameter.listIdentifiersResumption) match {
case Seq() => ListIdentifiersExclusiveRunner(CheckedListIdentifiersParameterExclusive(verb.get
,rt
,allParameter))
case setWithIllegalItems => BadArgumentsErrorRunner(BadArgumentsParameter(verb.get
,setWithIllegalItems.map(key => (key, key)).toMap
,allParameter))
}
case Failure(exceptionRT) =>
BadArgumentsErrorRunner(BadArgumentsParameter(verb.get, Map("resumptionToken" -> exceptionRT.getMessage)
ESResumptionTokenHelper.createResumptionTokenFromOaiToken(resumptionToken) match {
case Success(rt) =>
checkParameterByName(allParameter, AllowedParameter.listIdentifiersResumption) match {
case Seq() =>
ListIdentifiersExclusiveRunner(CheckedListIdentifiersParameterExclusive(
verb.get
,rt
,allParameter))
case setWithIllegalItems => BadArgumentsErrorRunner(BadArgumentsParameter(verb.get
, setWithIllegalItems.map(key => (key, key)).toMap
, allParameter))
}
//todo: make it right - only for completion here
case OaiCheckedVerbWithParameter(verb, _, _, _, _, _, _, allParameter,_ ) =>
//todo: ich muss noch einen Weg finden herauszufinden, welche Kombination dieser Parameter dazu führt, dass diese
// Kombination nicht erlaubt ist
val setWithIllegalItems = checkParameterByName(allParameter, AllowedParameter.listIdentifiersAllAllowed)
//todo: is this correct?
val setWithMissingItems = checkMissingParameterByName(AllowedParameter.listIdentifiersAllAllowed, allParameter)
BadArgumentsErrorRunner(BadArgumentsParameter(verb.get, setWithIllegalItems.map(key => (key, key)).toMap
, allParameter))
case Failure(exceptionRT) =>
BadArgumentsErrorRunner(BadArgumentsParameter(verb.get,
Map("resumptionToken" -> exceptionRT.getMessage),
allParameter))
}
case OaiCheckedVerbWithParameter(verb, _, _, _, _, _, _, allParameter, configuration) =>
//todo: das nochmals anschauen
BadArgumentsErrorRunner(BadArgumentsParameter(verb.get,
Map("illegalParameter" -> "no valid parameter"),
allParameter))
}
......
......@@ -113,7 +113,7 @@ object ListRecordsValidation extends ParameterValidationFunction {
Map("metadataPrefix" -> s"$metadataPrefix no valid specification"),
allParameter))
} else {
ListRecordsRunner(CheckedListRecordsParameter(verb, metadataPrefix, from, until, Some(set)
ListRecordsRunner(CheckedListRecordsParameter(verb, metadataPrefix, from, until, None
, allParameter))
}
......@@ -145,19 +145,14 @@ object ListRecordsValidation extends ParameterValidationFunction {
case OaiCheckedVerbWithParameter(Some(verb), _, _, _, _, _, _, allParameter, configuration) => ???
case OaiCheckedVerbWithParameter(verb, _, _, _, _, _, _, allParameter, configuration) =>
//todo: das nochmals anschauen
BadArgumentsErrorRunner(BadArgumentsParameter(verb.get,
Map("illegalParameter" -> "no valid parameter"),
allParameter))
/*
checkParameterByName(allParameter, AllowedParameter.listRecordsAllAllowed) match {
case setWithIllegalItems => BadArgumentsReq(BadArgumentsParameter(verb.get
, setWithIllegalItems.map(key => (key, key)).toMap
, allParameter))
}
*/
}
}
......@@ -47,22 +47,26 @@ object ListSetsValidation extends ParameterValidationFunction {
// es braucht dafür aber ein Konzept
case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, None, allParameter, configuration) =>
case OaiCheckedVerbWithParameter(Some(verb), _, _, _, _, _, _, allParameter, configuration) =>
checkParameterByName(allParameter, AllowedParameter.listSetsAllowed) match {
case Seq() => ListSetsRunner(CheckedListSetsParameter(verb.get
, allParameter))
case setWithIllegalItems => BadArgumentsErrorRunner(BadArgumentsParameter(verb.get
case Seq() => ListSetsRunner(CheckedListSetsParameter(verb, allParameter))
case setWithIllegalItems => BadArgumentsErrorRunner(BadArgumentsParameter(verb
, setWithIllegalItems.map(key => (key, key)).toMap
, allParameter))
}
/*
case OaiCheckedVerbWithParameter(verb, None, None, None, None, None, _, allParameter, configuration) =>
checkParameterByName(allParameter, AllowedParameter.listSetsAllowed) match {
case setWithIllegalItems => BadArgumentsErrorRunner(BadArgumentsParameter(verb.get
, setWithIllegalItems.map(key => (key, key)).toMap
, allParameter))
}
*/
//only as fallack - otherwise code doesn't compile
case OaiCheckedVerbWithParameter(verb, _, _, _, _, _, _, allParameter, configuration) =>
//todo: das nochmals anschauen
......@@ -73,6 +77,7 @@ object ListSetsValidation extends ParameterValidationFunction {
}
}
}
\ No newline at end of file
......@@ -60,7 +60,7 @@ abstract class ParameterValidationFunction () extends Function1[OaiCheckedVerbWi
allowedList.toSet.diff(paramsInReq.keySet).toSeq
}
def checkValidSet(set: String, oaiConfig: OaiConfig): Boolean = oaiConfig.sets.exists(_.spec == set)
def checkValidSet(set: String, oaiConfig: OaiConfig): Boolean = oaiConfig.sets.contains(set)
def checkValidMetadaPrefix(metadataPrefix: String, oaiConfig: OaiConfig): Boolean = oaiConfig.prefixes
......
......@@ -53,7 +53,7 @@ object OaiVerb extends Enumeration {
def getVerb(verb: Option[String]): OaiVerb = {
Try[OaiVerb] (OaiVerb.withName(verb.get)) match {
case Success(value) =>
print(value)
//print(value)
value
case Failure(exception) => if (mv.matches(exception.getMessage)) MissingVerb else WrongVerb
......
......@@ -54,7 +54,8 @@ case class ListRecordsResponse(config: Configuration
resultList.result.map(hit =>
<record>
{makeHeader(identifier = hit.docId,
datestamp = Instant.parse(hit.updateDate))}
datestamp = Instant.parse(hit.updateDate)
)}
<metadata>{XML.loadString(hit.document)}</metadata>
</record>)
......
......@@ -48,10 +48,10 @@ case class ListSetsResponse(config: Configuration
override def transform(n: Node): Seq[Node] = n match {
case elem: Elem if elem.label == "request" =>
elem ++ <ListSets>
{sets.map(prefixDef =>
{sets.map(setDefItem =>
<set>
<setSpec>{prefixDef.spec}</setSpec>
<setName>{prefixDef.name}</setName>
<setSpec>{setDefItem._2.spec}</setSpec>
<setName>{setDefItem._2.name}</setName>
</set>
)}
</ListSets>
......