Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

Commit 48b57566 authored by Günter Hipler's avatar Günter Hipler
Browse files

more error handling / use env variables for configuration

parent e83bae7f
Pipeline #23674 passed with stage
in 4 minutes and 10 seconds
......@@ -46,13 +46,16 @@ class ElasticsearchClient @Inject()(
Future.successful(client.getOrElse(Option.empty))
})
override val index: String = config.getString("elasticsearch.index")
override val index: String = getEnv("ELASTICSEARCH_INDEX")
//val client: Option[RestHighLevelClient] = connect()
override val client: Option[RestHighLevelClient] = connect()
//val oai = loadOaiConfig()
override def listLength: Int = getEnv("RESPONSE_LISTLENGTH").toIntOption.getOrElse(30)
override def resumptionTokenTTL: Long = getEnv("RESUMPTION_TOKEN_TTL").toLongOption.getOrElse(3)
override val oaiConfig: OaiConfig = loadOaiConfig()
//uploadTemplates()
......@@ -91,16 +94,29 @@ class ElasticsearchClient @Inject()(
private def connect(): Option[RestHighLevelClient] = {
val hosts = new ArrayBuffer[HttpHost]
config.getStringList("elasticsearch.hosts").forEach(
val configuredHosts = getEnv("ELASTICSEARCH_HOSTS").split(";")
val configuredPort = getEnv("ELASTICSEARCH_PORT").toIntOption.getOrElse(8080)
configuredHosts.foreach(
value => {
val hostPort = value.split(":")
hosts += new HttpHost(hostPort(0), hostPort(1).toInt)
hosts += new HttpHost(value, configuredPort)
}
)
val headers = Array(new BasicHeader("cluster.name", config.getString("elasticsearch.cluster")).asInstanceOf[Header])
val headers = Array(new BasicHeader("cluster.name", getEnv("ELASTICSEARCH_CLUSTER"))
.asInstanceOf[Header])
Option(new RestHighLevelClient(RestClient.builder(hosts.toArray : _*).setDefaultHeaders(headers)))
}
private def getEnv(value: String): String = {
val envValue: Option[String] = sys.env.get(value)
if (envValue.isDefined) {
envValue.get
} else {
Option(System.getProperties.get(value)) match {
case Some(value) => value.toString
case None => throw new Exception("Environment variable " + value + " not available")
}
}
}
}
......@@ -33,6 +33,8 @@ import org.elasticsearch.search.builder.SearchSourceBuilder
import org.swissbib.memobase.oai.common.util.{ESResumptionTokenHelper, ResumptionToken}
import org.swissbib.memobase.oai.runner.{GetRecordFailure, ResultList}
import play.Environment
import utilities.Helper
import scala.jdk.CollectionConverters._
import scala.util.{Failure, Success, Try}
......@@ -60,8 +62,9 @@ trait ElasticsearchComponent extends OaiRepository {
.lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
val searchRequest = new SearchRequest().source(searchSourceBuilder)
.indices(index).scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
......@@ -70,8 +73,9 @@ trait ElasticsearchComponent extends OaiRepository {
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.gte(from).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index)
.scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
......@@ -80,8 +84,9 @@ trait ElasticsearchComponent extends OaiRepository {
val rqB = QueryBuilders
.rangeQuery("lastUpdatedDate")
.lte(until).format("strict_date_time")
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
val searchSourceBuilder = new SearchSourceBuilder().query(rqB).size(listLength)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index)
.scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
......@@ -93,8 +98,9 @@ trait ElasticsearchComponent extends OaiRepository {
//only for the moment until we have better data
case (_, _,_,_,_) =>
//for the moment
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(30)
val searchRequest = new SearchRequest().source(searchSourceBuilder).indices(index).scroll(TimeValue.timeValueMinutes(3L))
val searchSourceBuilder = new SearchSourceBuilder().query(QueryBuilders.matchAllQuery()).size(listLength)
val searchRequest = new SearchRequest().source(searchSourceBuilder)
.indices(index).scroll(TimeValue.timeValueMinutes(resumptionTokenTTL))
Try {client.get.search(searchRequest, RequestOptions.DEFAULT)}
......@@ -108,16 +114,18 @@ trait ElasticsearchComponent extends OaiRepository {
//condition to finish the fetching is to compare the length of the resultlist
//compare: https://www.elastic.co/guide/en/elasticsearch/client/java-api/current/java-search-scrolling.html
//otherwise we will get into an empty loop
val scrollId = if (searchResponse.getHits.getHits.length < 30) None else Option(searchResponse.getScrollId)
val scrollId = if (searchResponse.getHits.getHits.length < listLength) None else Option(searchResponse.getScrollId)
val resumptionToken: Option[ResumptionToken] = scrollId.map(
ESResumptionTokenHelper(_))
val contentList = searchResponse.getHits.getHits.
map(hit => {
val source = hit.getSourceAsMap.asScala
OAIContent(hit.getId,
source.getOrElse("id","").toString,
source.getOrElse("document","").toString,
if (source.contains("document")) (Helper.decodeBase64FromString(source("document").toString)) else "",
//source.getOrElse("document","").toString,
source.getOrElse("format","").toString,
source.getOrElse("published",false).asInstanceOf[Boolean],
source.getOrElse("recordset",new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
......@@ -161,7 +169,9 @@ trait ElasticsearchComponent extends OaiRepository {
getResponse.getId
OAIContent(getResponse.getId,
hit.getOrElse("id", "").toString,
hit.getOrElse("document", "").toString,
//todo: test it
if (hit.contains("document")) (Helper.decodeBase64FromString(hit("document").toString)) else "",
//hit.getOrElse("document", "").toString,
hit.getOrElse("format", "").toString,
hit.getOrElse("published", false).asInstanceOf[Boolean],
hit.getOrElse("recordset", new util.ArrayList[String]()).asInstanceOf[util.ArrayList[String]].asScala.toList,
......
......@@ -52,6 +52,12 @@ trait OaiRepository {
metadataPrefix: String
): Try[OAIContent]
def listLength: Int
def resumptionTokenTTL: Long
}
......@@ -109,6 +109,12 @@ case class NoContentForQueryParameter (verb: OaiVerb
,errorCode: String ) extends OaiParameterBase
case class SystemErrorParameter (verb: OaiVerb
,allQueryParameter: Map[String, Seq[String]]
,errorCode: String
,errorMessage: String) extends OaiParameterBase
object OAIErrorCode extends Enumeration {
......@@ -124,6 +130,7 @@ object OAIErrorCode extends Enumeration {
val NO_RECORDS_MATCH: OAIErrorCodeVal = OAIErrorCodeVal("noRecordsMatch")
val ID_DOES_NOT_EXIST: OAIErrorCodeVal = OAIErrorCodeVal("idDoesNotExist")
val SYSTEM_ERROR: OAIErrorCodeVal = OAIErrorCodeVal("idSystemError")
def getErrorCode(code: Option[String]): Try[OAIErrorCode] = {
......
......@@ -58,7 +58,7 @@ case class ListIdentifiersResponse( config: Configuration
)
}
{if (resultList.repositoryToken.isDefined) {
<resumptionToken>{resultList.repositoryToken}</resumptionToken>
<resumptionToken>{resultList.repositoryToken.get.token}</resumptionToken>
}}
</ListIdentifiers>
......
......@@ -37,15 +37,9 @@ abstract class OaiResponse (oaiConfig: OaiConfig
datestamp: Instant = Instant.now(),
identifier: String): Elem =
<header status={if (del) Some(Text("deleted")) else None}>
<identifier>
{identifier}
</identifier>
<datestamp>
{datestamp}
</datestamp>
<setSpec>
{set}
</setSpec>
<identifier>{identifier}</identifier>
<datestamp>{datestamp}</datestamp>
<setSpec>{set}</setSpec>
</header>
......
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package org.swissbib.memobase.oai.response
import com.typesafe.config.ConfigObject
import modules.OaiRepository
import org.swissbib.memobase.oai.common.validation.{BadArgumentsParameter, SystemErrorParameter}
import play.api.Configuration
import scala.collection.immutable
import scala.xml.transform.{RewriteRule, RuleTransformer}
import scala.xml.{Elem, Node}
case class SystemErrorResponse(config: Configuration
, repository: OaiRepository
, parameter: SystemErrorParameter) extends OaiResponse (repository.oaiConfig
,parameter) {
override def toString: String = s"Ich bin eine BadArguments response "
override def createResponse: Seq[Node] = {
//todo create the body using the executed action in repository
val oaiFrame = createOaiFrame
val sets: Seq[ConfigObject] = repository.oaiConfig.sets.sets
//val configTuples: Seq[(ConfigValue, ConfigValue)] = sets.map(f => (f.get("spec"),f.get("name")))
val verbBody: RewriteRule = new RewriteRule {
override def transform(n: Node): Seq[Node] = n match {
case elem: Elem if elem.label == "request" =>
elem ++ {
<error code={parameter.errorCode}>{parameter.errorMessage}</error>
}
case n => n
}
}
new RuleTransformer(verbBody).transform(oaiFrame)
}
}
......@@ -23,7 +23,7 @@
package org.swissbib.memobase.oai.runner
import modules.{OAIContent, OaiRepository}
import org.swissbib.memobase.oai.common.validation.{CheckedGetRecordParameter, NoContentForQueryParameter, OAIErrorCode}
import org.swissbib.memobase.oai.common.validation.{CheckedGetRecordParameter, NoContentForQueryParameter, OAIErrorCode, SystemErrorParameter}
import org.swissbib.memobase.oai.response.{GetRecordResponse, OaiResponse}
import play.api.Configuration
......@@ -38,11 +38,16 @@ case class GetRecordRunner(parameter: CheckedGetRecordParameter) extends OaiRunn
GetRecordResponse(config, repository, result,parameter)
case Failure(exception) =>
exception match {
case _: GetRecordFailure => NoContentForQueryRunner(NoContentForQueryParameter(parameter.verb
case e: GetRecordFailure =>
NoContentForQueryRunner(NoContentForQueryParameter(parameter.verb
, parameter.allQueryParameter
, OAIErrorCode.ID_DOES_NOT_EXIST)).run()
//todo: make something useful for system errors
case _ => ???
case ee =>
SystemErrorRunner(SystemErrorParameter(parameter.verb,parameter.allQueryParameter,
OAIErrorCode.SYSTEM_ERROR,exception.getMessage)).run()
}
}
}
......
......@@ -23,13 +23,12 @@
package org.swissbib.memobase.oai.runner
import modules.OaiRepository
import org.swissbib.memobase.oai.common.validation.{CheckedListIdentifiersParameter, CheckedListIdentifiersParameterExclusive, NoContentForQueryParameter, OAIErrorCode}
import org.swissbib.memobase.oai.common.validation.{CheckedListIdentifiersParameter, CheckedListIdentifiersParameterExclusive, NoContentForQueryParameter, OAIErrorCode, SystemErrorParameter}
import org.swissbib.memobase.oai.response.{ListIdentifiersResponse, ListRecordsResponse, OaiResponse}
import play.api.Configuration
import scala.util.{Failure, Success}
//CheckedListIdentifiersParameter
case class ListIdentifiersRunner(parameter: CheckedListIdentifiersParameter) extends OaiRunner {
override def run()(implicit config: Configuration, repository: OaiRepository): OaiResponse = {
......@@ -46,15 +45,11 @@ case class ListIdentifiersRunner(parameter: CheckedListIdentifiersParameter) ext
case Nil => NoContentForQueryRunner(NoContentForQueryParameter(parameter.verb
, parameter.allQueryParameter
,OAIErrorCode.NO_RECORDS_MATCH)).run()
//at least one result
case _ => ListIdentifiersResponse(config,repository,requestResult, parameter)
}
case Failure(exception) => ???
//todo: make something useful
//ListRecordsResponse(config,repository,null, repository.oaiConfig,parameter)
case Failure(exception) =>
SystemErrorRunner(SystemErrorParameter(parameter.verb,parameter.allQueryParameter,
OAIErrorCode.SYSTEM_ERROR,exception.getMessage)).run()
}
......
......@@ -25,7 +25,7 @@
package org.swissbib.memobase.oai.runner
import modules.OaiRepository
import org.swissbib.memobase.oai.common.validation.{CheckedListRecordsParameter, CheckedListRecordsParameterExclusive, NoContentForQueryParameter, OAIErrorCode}
import org.swissbib.memobase.oai.common.validation.{CheckedListRecordsParameter, CheckedListRecordsParameterExclusive, NoContentForQueryParameter, OAIErrorCode, SystemErrorParameter}
import org.swissbib.memobase.oai.response.{ListRecordsResponse, OaiResponse}
import play.api.Configuration
......@@ -51,8 +51,9 @@ case class ListRecordsRunner(parameter: CheckedListRecordsParameter) extends Oai
//at least one result
case _ => ListRecordsResponse(config,repository,requestResult, parameter)
}
case Failure(exception) => ???
//ListRecordsResponse(config,repository,null, repository.oaiConfig,parameter)
case Failure(exception) =>
SystemErrorRunner(SystemErrorParameter(parameter.verb,parameter.allQueryParameter,
OAIErrorCode.SYSTEM_ERROR,exception.getMessage)).run()
}
}
}
......
......@@ -23,8 +23,8 @@
package org.swissbib.memobase.oai.runner
import modules.OaiRepository
import org.swissbib.memobase.oai.common.validation.{BadArgumentsParameter, NoContentForQueryParameter}
import org.swissbib.memobase.oai.response.{BadArgumentsResponse, NoContentForQueryResponse, OaiResponse}
import org.swissbib.memobase.oai.common.validation.{BadArgumentsParameter, NoContentForQueryParameter, SystemErrorParameter}
import org.swissbib.memobase.oai.response.{BadArgumentsResponse, NoContentForQueryResponse, OaiResponse, SystemErrorResponse}
import play.api.Configuration
......@@ -56,4 +56,12 @@ case class BadArgumentsErrorRunner(parameter: BadArgumentsParameter) extends Oai
BadArgumentsResponse(config, repository,parameter)
}
case class SystemErrorRunner(parameter: SystemErrorParameter) extends OaiRunner {
override def run()(implicit config: Configuration, repository: OaiRepository): OaiResponse =
SystemErrorResponse(config, repository,parameter)
}
/*
* generic OAI Server - agnostic in relation to the data repository
* initially created for memobase project
*
* Copyright (C) 2021 UB Basel
*
* This program is free software: you can redistribute it and/or modify
* it under the terms of the GNU Affero General Public License as
* published by the Free Software Foundation, either version 3 of the
* License, or (at your option) any later version.
*
* This program is distributed in the hope that it will be useful,
* but WITHOUT ANY WARRANTY; without even the implied warranty of
* MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
* GNU Affero General Public License for more details.
*
* You should have received a copy of the GNU Affero General Public License
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*
*/
package utilities
import java.io.ByteArrayOutputStream
import java.util.Base64
import java.util.zip.Inflater
object Helper {
def deserialize(data: Array[Byte]): String = {
val decompressor = new Inflater()
decompressor.setInput(data)
val bos = new ByteArrayOutputStream(data.length)
val buffer = new Array[Byte](8192)
while (!decompressor.finished) {
val size = decompressor.inflate(buffer)
bos.write(buffer, 0, size)
}
bos.toString
}
def decodeBase64FromString(encodedBase64String: String): String = {
deserialize(Base64.getDecoder.decode(encodedBase64String))
}
}
......@@ -19,6 +19,7 @@ lazy val root = (project in file("."))
assemblyJarName in assembly := "app.jar",
//mainClass in assembly := Some("ch.memobase.rico2edm.Main"),
test in assembly := {},
/*
assemblyMergeStrategy in assembly := {
case "log4j.properties" => MergeStrategy.first
case "module-info.class" => MergeStrategy.discard
......@@ -36,7 +37,7 @@ lazy val root = (project in file("."))
oldStrategy(x)
},
*/
//assemblyJarName in assembly := "app.jar",
//test in assembly := {},
......
......@@ -19,21 +19,6 @@ repository.dispatcher {
}
elasticsearch {
//template_prefix="production"
//hosts: ["sb-ues5.swissbib.unibas.ch:8080","sb-ues6.swissbib.unibas.ch:8080","sb-ues7.swissbib.unibas.ch:8080","sb-ues8.swissbib.unibas.ch:8080"]
hosts: ["mb-es1.memobase.unibas.ch:8080"]
//hosts: ["localhost:8080"]
cluster: "test-memobase-search-cluster" # "weywot"
index: "oai-v1"
content: "application/json; charset=utf-8"
#the default number of result per page
defaultsize: 10
#the maximum number of result per page you can request (without scroll)
maxsize: 1000
}
kafka : [
{
......
......@@ -20,10 +20,23 @@ spec:
containers:
- name: oai-api-prod-container
image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/externalapis/oai:develop
#image: image: guenterh/oaimemobase:v0.5
#image: cr.gitlab.switch.ch/memoriav/memobase-2020/services/externalapis/oai:latest
image: guenterh/oaimemobase:0.5.3
ports:
- containerPort: 9000
name: http
protocol: TCP
imagePullPolicy: Always
env:
- name: RESPONSE_LISTLENGTH
value: "30"
- name: RESUMPTION_TOKEN_TTL
value: "3"
- name: ELASTICSEARCH_INDEX
value: oai-v2
- name: ELASTICSEARCH_HOSTS
value: localhost
- name: ELASTICSEARCH_PORT
value: "8080"
- name: ELASTICSEARCH_CLUSTER
value: test-memobase-search-cluster
\ No newline at end of file
RESPONSE_LISTLENGTH=30
RESUMPTION_TOKEN_TTL=3
ELASTICSEARCH_INDEX=oai-v2
ELASTICSEARCH_HOSTS=localhost
ELASTICSEARCH_PORT=8080
ELASTICSEARCH_CLUSTER=test-memobase-search-cluster
......@@ -23,7 +23,7 @@ object Dependencies {
lazy val joda = "org.joda" % "joda-convert" % "2.2.1"
lazy val logging = "net.logstash.logback" % "logstash-logback-encoder" % "6.6"
lazy val logging = "net.logstash.logback" % "logstash-logback-encoder" % "6.2"
lazy val lemonlabs = "io.lemonlabs" %% "scala-uri" % "1.5.1"
lazy val codingwell = "net.codingwell" %% "scala-guice" % "4.2.6"
//"org.apache.solr" % "solr-solrj" % "7.3.1",
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment