Skip to content

GitLab

Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

Commit 1f0b63c2 authored by Günter Hipler's avatar Günter Hipler
Browse files

Merge branch 'develop'

parents df2fd457 b8119b8c
Hide whitespace changes
Inline Side-by-side
app/controllers/OaiController.scala
View file @ 1f0b63c2
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package controllers
import javax.inject.Inject
import modules.OaiRepository
import org.swissbib.memobase.oai.request.OaiRequest
import org.swissbib.memobase.oai.common.verb.OaiVerb
import org.swissbib.memobase.oai.common.validation.OaiQueryParameterChecker
import org.swissbib.memobase.oai.runner.OaiRunner
import play.api.Configuration
import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponents, Rendering}
class OaiController @Inject()(cc: ControllerComponents,
config: Configuration,
repository:OaiRepository
) extends AbstractController(cc) with Rendering {
class OaiController @Inject()(cc: ControllerComponents)
(implicit config: Configuration,repository:OaiRepository ) extends AbstractController(cc) with Rendering {
//todo: better way might be to use a so called BindController - have a look into this type part of this package
// where I played around a little bit with it
//https://www.playframework.com/documentation/2.8.x/RequestBinders
def handleRequest(verb: String,
//todo: can we make the Action more type specific (not AnyContent)
def handleRequest(verb: Option[String],
metadataPrefix: Option[String],
set: Option[String],
from:Option[String],
......@@ -25,27 +42,21 @@ class OaiController @Inject()(cc: ControllerComponents,
resumptionToken: Option[String]): Action[AnyContent] =
Action { implicit request =>
//Todo has to be wrapped with Try because verb might be wrong
val enumVerb = OaiVerb.withName(verb)
val request = OaiRequest(enumVerb,
metadataPrefix,
set,
from,
until,
identifier,
resumptionToken
)
val response = request.execute(config, repository).createResponse
/*
todo: make Writable for Node[Seq]
[info] Compiling 4 Scala sources to /home/swissbib/environment/code/swissbib.repositories/memoriav/gitlab/services/oai/target/scala-2.13/classes ...
[error] /home/swissbib/environment/code/swissbib.repositories/memoriav/gitlab/services/oai/app/controllers/OaiController.scala:44:9: Cannot write an instance of Seq[scala.xml.Node] to HTTP response. Try to define a Writeable[Seq[scala.xml.Node]]
[error] Ok(response.createResponse).as("text/xml")
*/
Ok(response.toString()).as("text/xml")
val qP: Map[String, Seq[String]] = request.queryString
val runner: OaiRunner = new OaiQueryParameterChecker(
verb = verb,
metadataPrefix = metadataPrefix,
set = set,
from = from,
until = until,
identifier = identifier,
resumptionToken = resumptionToken,
allQueryParameter = qP,
configuration = config).checkQueryParameter
val response = runner.run().createResponse.toString()
Ok(response).as("text/xml")
}
def fallback: Action[AnyContent] =
......
app/modules/ElasticsearchClient.scala
View file @ 1f0b63c2
package modules
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
import java.io.IOException
import java.util
import com.typesafe.config.{Config, ConfigObject}
package modules
import com.typesafe.config.Config
import javax.inject.{Inject, Singleton}
import org.apache.http.{Header, HttpHost}
import org.apache.http.message.BasicHeader
import org.elasticsearch.action.admin.cluster.storedscripts.PutStoredScriptRequest
import org.elasticsearch.client.{RequestOptions, RestClient, RestHighLevelClient}
import org.elasticsearch.common.bytes.BytesReference
import org.elasticsearch.common.xcontent.{XContentFactory, XContentType}
import org.elasticsearch.client.{RestClient, RestHighLevelClient}
import org.swissbib.memobase.oai.common.util.{OaiCommonConfig, OaiConfig, OaiConfigMetadataPrefixes, OaiConfigSets, OaiIdentifyConfig}
import play.Environment
import play.api.inject.ApplicationLifecycle
import utilities.FileUtil
import scala.jdk.CollectionConverters._
import scala.collection.mutable.ArrayBuffer
import scala.concurrent.Future
@Singleton
class ElasticsearchClient @Inject()(
lifecycle: ApplicationLifecycle,
private val config: Config,
private val env: Environment
val config: Config,
val env: Environment
) extends ElasticsearchComponent {
lifecycle.addStopHook(() => {
//Future.successful(client.get.close())
Future.successful(client.getOrElse(Option.empty))
})
override val index: String = config.getString("elasticsearch.index")
//val client: Option[RestHighLevelClient] = connect()
override val client: Option[RestHighLevelClient] = None
override val client: Option[RestHighLevelClient] = connect()
//val oai = loadOaiConfig()
override val oaiConfig: OaiConfig = loadOaiConfig()
//uploadTemplates()
......@@ -73,41 +91,16 @@ class ElasticsearchClient @Inject()(
private def connect(): Option[RestHighLevelClient] = {
val hosts = new ArrayBuffer[HttpHost]
config.getStringList("index.hosts").forEach(
config.getStringList("elasticsearch.hosts").forEach(
value => {
val hostPort = value.split(":")
hosts += new HttpHost(hostPort(0), hostPort(1).toInt)
}
)
val headers = Array(new BasicHeader("cluster.name", config.getString("index.cluster")).asInstanceOf[Header])
val headers = Array(new BasicHeader("cluster.name", config.getString("elasticsearch.cluster")).asInstanceOf[Header])
Option(new RestHighLevelClient(RestClient.builder(hosts.toArray : _*).setDefaultHeaders(headers)))
}
private def uploadTemplates(): Unit = {
config.getStringList("index.templatequeries").forEach((templateName: String) => {
val template = FileUtil.readFile(templateName, env)
val nameWithoutPath = templateName.substring(templateName.lastIndexOf("/") + 1).replaceAll(".mustache", "")
val templatePrefix = config.getString("index.template_prefix")
try {
val request = new PutStoredScriptRequest
request.id(templatePrefix + "_" + nameWithoutPath)
val builder = XContentFactory.jsonBuilder
builder.startObject
builder.startObject("script")
builder.field("lang", "mustache")
// load mustache source as string to allow invalid JSON template features!
builder.field("source", template.replace("\n", ""))
builder.endObject
builder.endObject
request.content(BytesReference.bytes(builder), XContentType.JSON)
client.get.putScript(request, RequestOptions.DEFAULT)
} catch {
case io: IOException =>
//TODO: if the templates cannot be read the application should shut down.
io.printStackTrace()
}
})
}
}
app/modules/ElasticsearchComponent.scala
View file @ 1f0b63c2
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import org.elasticsearch.client.RestHighLevelClient
import java.util
import com.typesafe.config.Config
import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.action.search.{SearchRequest, SearchResponse, SearchScrollRequest}
import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilders
import org.elasticsearch.search.builder.SearchSourceBuilder
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
import org.swissbib.memobase.oai.runner.{GetRecordFailure, ResultList}
import play.Environment
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}
trait ElasticsearchComponent extends OaiRepository {
val client: Option[RestHighLevelClient]
val config: Config
val env: Environment
//to do: another solution necessary in case we decide to use several indices for OAI
val index: String
override def listRecords(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Try[ResultList] = Try{
//todo - more / better / tested handling for different parameters
val searchResponse:Try[SearchResponse] = (from, until, set, resumptionToken, metadataPrefix) match {
case (Some(from), Some(until),_,None,_) =>
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
.lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (Some(from), None ,_,None,_) =>
//for the moment
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (None, Some(until) ,_,None,_) =>
//for the moment
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
case (_, _,_,Some(resumptionToken),_) =>
val scrollRequest = new SearchScrollRequest(resumptionToken.subject)
scrollRequest.scroll(TimeValue.timeValueMinutes(3L))
Try[SearchResponse] {client.get.scroll(scrollRequest, RequestOptions.DEFAULT)}
//only for the moment until we have better data
case (_, _,_,_,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
}
searchResponse match {
case Success(searchResponse) =>
//scroll_id is always the same -> for this specific context
//condition to finish the fetching is to compare the length of the resultlist
//compare: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html
//otherwise we will get into an empty loop
val scrollId = if (searchResponse.getHits.getHits.length < 30) None else Option(searchResponse.getScrollId)
val resumptionToken: Option[ResumptionToken] = scrollId.map(
ESResumptionTokenHelper(_))
val contentList = searchResponse.getHits.getHits.
map(hit => {
val source = hit.getSourceAsMap.asScala
OAIContent(hit.getId,
source.getOrElse("id","").toString,
source.getOrElse("document","").toString,
source.getOrElse("format","").toString,
source.getOrElse("published",false).asInstanceOf[Boolean],
source.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
source.getOrElse("institution",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
source.getOrElse("lastUpdatedDate", "").asInstanceOf[String])
}
).toSeq
ResultList(resumptionToken, contentList)
case Failure(exception) => throw exception
}
}
override def listIdentiers(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Try[ResultList] =
listRecords(from,
until,
set,
resumptionToken,
metadataPrefix)
override def getRecord(identifier: String,
metadataPrefix: String): Try[OAIContent] = Try {
val getRequest = new GetRequest(index, identifier)
val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT)
if (getResponse.isSourceEmpty) {
throw GetRecordFailure("record was not found")
} else {
val hit = getResponse.getSource.asScala
getResponse.getId
OAIContent(getResponse.getId,
hit.getOrElse("id", "").toString,
hit.getOrElse("document", "").toString,
hit.getOrElse("format", "").toString,
hit.getOrElse("published", false).asInstanceOf[Boolean],
hit.getOrElse("recordset", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
hit.getOrElse("institution", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
hit.getOrElse("lastUpdatedDate", "").asInstanceOf[String]
)
}
}
}
app/modules/ElasticsearchModule.scala
View file @ 1f0b63c2
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import com.google.inject.AbstractModule
import com.google.inject.AbstractModule
class ElasticsearchModule extends AbstractModule{
override def configure(): Unit = {
/*
bind(classOf[ElasticsearchComponent])
.to(classOf[ElasticsearchClient])
.asEagerSingleton()
*/
bind(classOf[OaiRepository])
.to(classOf[ElasticsearchClient])
.asEagerSingleton()
......
app/modules/KafkaComponent.scala
View file @ 1f0b63c2
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import com.typesafe.config.Config
......@@ -14,4 +36,5 @@ class KafkaComponentImpl @Inject()(lifecycle: ApplicationLifecycle,
config: Config,
environment: Environment) extends KafkaComponent {
}
app/modules/KafkaModule.scala
View file @ 1f0b63c2
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import com.google.inject.AbstractModule
......@@ -20,6 +42,20 @@ class KafkaModule(environment: Environment, configuration: Configuration)
//bind[SolrComponent].to[SolrComponentImpl].in[Singleton]
bind[KafkaComponent].to[KafkaComponentImpl].in[Singleton]
//val t1 = environment
//val t2 = configuration
//bind[KafkaComponent].to[KafkaComponentImpl].in[Singleton]
}
}
/*
Fehler in 2.13.3
[info] Non-compiled module 'compiler-bridge_2.13' for Scala 2.13.3. Compiling...
[info] Compilation completed in 8.085s.
[error] /home/swissbib/environment/code/repositories/memoriav/gitlab/services/externalapis/oai/app/modules/KafkaModule.scala:25:51: Auto-application to `()` is deprecated. Supply the empty argument list `()` explicitly to invoke method in,
[error] or remove the empty argument list from its definition (Java-defined methods are exempt).
[error] In Scala 3, an unapplied method like this will be eta-expanded into a function.
[error] bind[KafkaComponent].to[KafkaComponentImpl].in[Singleton]
*/
app/modules/OaiRepository.scala
View file @ 1f0b63c2
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package modules
import org.swissbib.memobase.oai.common.util.OaiConfig
import org.swissbib.memobase.oai.common.util.{OaiConfig, ResumptionToken}
import org.swissbib.memobase.oai.runner.ResultList
import scala.util.Try
case class OAIContent(esId: String, docId: String, document: String, format: String, published: Boolean,set: List[String], institution: List[String], updateDate: String )
trait OaiRepository {
val oaiConfig: OaiConfig
def listRecords(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Try[ResultList]
def listIdentiers(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Try[ResultList]
def getRecord(identifier: String,
metadataPrefix: String
): Try[OAIContent]
}
app/org/swissbib/memobase/oai/common/util/OaiConfig.scala
View file @ 1f0b63c2
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package org.swissbib.memobase.oai.common.util
import com.typesafe.config.ConfigObject
......
app/org/swissbib/memobase/oai/common/util/RecordHeader.scala deleted 100644 → 0
View file @ df2fd457
package org.swissbib.memobase.oai.common.util
class RecordHeader {
}