Commit 89cef8b7 authored by Günter Hipler's avatar Günter Hipler
Browse files

GetRecord Implementation with ES

parent 774d760a
Pipeline #12268 passed with stage
in 2 minutes and 8 seconds
package controllers
import javax.inject.Inject
import modules.OaiRepository
import modules.{ElasticsearchComponent, OaiRepository}
import org.swissbib.memobase.oai.request.OaiRequest
import org.swissbib.memobase.oai.common.verb.OaiVerb
import play.api.Configuration
......@@ -9,7 +9,8 @@ import play.api.mvc.{AbstractController, Action, AnyContent, ControllerComponent
class OaiController @Inject()(cc: ControllerComponents,
config: Configuration,
repository:OaiRepository
repository:OaiRepository,
) extends AbstractController(cc) with Rendering {
......@@ -27,6 +28,7 @@ class OaiController @Inject()(cc: ControllerComponents,
//Todo has to be wrapped with Try because verb might be wrong
val enumVerb = OaiVerb.withName(verb)
val t = repository
val request = OaiRequest(enumVerb,
metadataPrefix,
......
......@@ -23,19 +23,22 @@ import scala.concurrent.Future
@Singleton
class ElasticsearchClient @Inject()(
lifecycle: ApplicationLifecycle,
private val config: Config,
private val env: Environment
val config: Config,
val env: Environment
) extends ElasticsearchComponent {
lifecycle.addStopHook(() => {
//Future.successful(client.get.close())
Future.successful(client.getOrElse(Option.empty))
})
override val index: String = config.getString("elasticsearch.index")
//val client: Option[RestHighLevelClient] = connect()
override val client: Option[RestHighLevelClient] = None
override val client: Option[RestHighLevelClient] = connect()
//val oai = loadOaiConfig()
override val oaiConfig: OaiConfig = loadOaiConfig()
//uploadTemplates()
......@@ -73,41 +76,16 @@ class ElasticsearchClient @Inject()(
private def connect(): Option[RestHighLevelClient] = {
val hosts = new ArrayBuffer[HttpHost]
config.getStringList("index.hosts").forEach(
config.getStringList("elasticsearch.hosts").forEach(
value => {
val hostPort = value.split(":")
hosts += new HttpHost(hostPort(0), hostPort(1).toInt)
}
)
val headers = Array(new BasicHeader("cluster.name", config.getString("index.cluster")).asInstanceOf[Header])
val headers = Array(new BasicHeader("cluster.name", config.getString("elasticsearch.cluster")).asInstanceOf[Header])
Option(new RestHighLevelClient(RestClient.builder(hosts.toArray : _*).setDefaultHeaders(headers)))
}
private def uploadTemplates(): Unit = {
config.getStringList("index.templatequeries").forEach((templateName: String) => {
val template = FileUtil.readFile(templateName, env)
val nameWithoutPath = templateName.substring(templateName.lastIndexOf("/") + 1).replaceAll(".mustache", "")
val templatePrefix = config.getString("index.template_prefix")
try {
val request = new PutStoredScriptRequest
request.id(templatePrefix + "_" + nameWithoutPath)
val builder = XContentFactory.jsonBuilder
builder.startObject
builder.startObject("script")
builder.field("lang", "mustache")
// load mustache source as string to allow invalid JSON template features!
builder.field("source", template.replace("\n", ""))
builder.endObject
builder.endObject
request.content(BytesReference.bytes(builder), XContentType.JSON)
client.get.putScript(request, RequestOptions.DEFAULT)
} catch {
case io: IOException =>
//TODO: if the templates cannot be read the application should shut down.
io.printStackTrace()
}
})
}
}
package modules
import org.elasticsearch.client.RestHighLevelClient
import com.typesafe.config.Config
import org.elasticsearch.action.get.GetRequest
import org.elasticsearch.client.{RequestOptions, RestHighLevelClient}
import org.swissbib.memobase.oai.common.util.ResumptionToken
import play.Environment
import play.api.libs.json.Json
import utilities.Json2XML
import scala.xml.Node
trait ElasticsearchComponent extends OaiRepository {
val client: Option[RestHighLevelClient]
val config: Config
val env: Environment
//to do: another solution necessary in case we decide to use several indices for OAI
val index: String
override def listRecords(from: Option[String], until: Option[String], set: Option[String], resumptionToken: Option[ResumptionToken], metadataPrefix: String): Seq[Node] = ???
override def listIdentiers(from: Option[String], until: Option[String], set: Option[String], resumptionToken: Option[ResumptionToken], metadataPrefix: String): Seq[Node] = ???
override def getRecord(identifier: String, metadataPrefix: String): Option[Node] = {
val getRequest = new GetRequest(index, identifier)
val getResponse = client.get.get(getRequest, RequestOptions.DEFAULT)
//todo: what if response is empty?
val sourceMap = getResponse.getSourceAsMap
Option(Json2XML.singleJsonDoc2Xml(Json.parse(play.libs.Json.toJson(sourceMap).toString)))
}
}
......@@ -14,4 +14,8 @@ class KafkaComponentImpl @Inject()(lifecycle: ApplicationLifecycle,
config: Config,
environment: Environment) extends KafkaComponent {
val t1: Config = config
val t2: Environment = environment
println("in kafka component")
}
......@@ -20,6 +20,8 @@ class KafkaModule(environment: Environment, configuration: Configuration)
//bind[SolrComponent].to[SolrComponentImpl].in[Singleton]
val t1 = environment
val t2 = configuration
bind[KafkaComponent].to[KafkaComponentImpl].in[Singleton]
}
}
package modules
import org.swissbib.memobase.oai.common.util.OaiConfig
import org.swissbib.memobase.oai.common.util.{OaiConfig, ResumptionToken}
import scala.xml.{Elem, Node}
trait OaiRepository {
val oaiConfig: OaiConfig
def listRecords(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Seq[Node]
def listIdentiers(from: Option[String],
until: Option[String],
set: Option[String],
resumptionToken: Option[ResumptionToken],
metadataPrefix: String): Seq[Node]
def getRecord(identifier: String,
metadataPrefix: String
): Option[Node]
}
......@@ -33,6 +33,8 @@ case class GetRecordRunner(override val config: Configuration,
override def run(): OaiResponse = {
//Identify doesnit need any action on repository
val node = repository.getRecord(request.identifier.get,request.metadataPrefix.get)
GetRecordResponse(this)
}
}
......
package utilities
import java.io.{ByteArrayInputStream, StringWriter}
import java.nio.charset.Charset
import de.odysseus.staxon.json.{JsonXMLConfigBuilder, JsonXMLInputFactory}
import de.odysseus.staxon.xml.util.PrettyXMLEventWriter
import javax.xml.stream.XMLOutputFactory
import play.api.libs.json.{JsObject, JsValue}
import scala.xml.Elem
object Json2XML {
def singleJsonDoc2Xml(doc:JsValue, root: String = "record"):Elem = {
//only Helper function until we know how and produce valid OAI data
val esJsonRespone = JsObject(Seq(root -> doc)).toString
val config = new JsonXMLConfigBuilder().multiplePI(false).build();
val output = new StringWriter()
val reader = new JsonXMLInputFactory(config).createXMLEventReader(
new ByteArrayInputStream( esJsonRespone.getBytes(Charset.forName("UTF-8")))
)
val writer = XMLOutputFactory.newInstance().createXMLEventWriter(output)
val prettyWriter = new PrettyXMLEventWriter(writer)
prettyWriter.add(reader)
reader.close()
writer.close()
scala.xml.XML.loadString(output.toString)
}
}
......@@ -31,7 +31,8 @@ lazy val root = (project in file("."))
codingwell,
scalatestplusplay % Test,
elasticsearch,
logging
logging,
json2xml
),
scalacOptions ++= Seq(
"-feature",
......
......@@ -4,6 +4,7 @@ include "secure"
play {
http.secret.key="Sl9aHk;Wb__x7ZblaWb4y]3;kcDPAr4>vsV[CC`;=]r1VWc=D8cSy4TBZ"
modules.enabled += "modules.ElasticsearchModule"
modules.enabled += "modules.KafkaModule"
}
# db connections = ((physical_core_count * 2) + effective_spindle_count)
......@@ -18,6 +19,20 @@ repository.dispatcher {
}
elasticsearch {
//template_prefix="production"
//hosts: ["sb-ues5.swissbib.unibas.ch:8080","sb-ues6.swissbib.unibas.ch:8080","sb-ues7.swissbib.unibas.ch:8080","sb-ues8.swissbib.unibas.ch:8080"]
hosts: ["localhost:8080"]
cluster: "test-memobase-search-cluster" # "weywot"
index: "documents-v6"
content: "application/json; charset=utf-8"
#the default number of result per page
defaultsize: 10
#the maximum number of result per page you can request (without scroll)
maxsize: 1000
}
kafka : [
{
......
......@@ -8,6 +8,7 @@ http://oai.swissbib.ch/oai/DB=2.1?verb=ListSets
ListIdentifiers
swissbib verlangt from was nicht korrekt ist
http://oai.swissbib.ch/oai/DB=2.1?verb=ListIdentifiers&metadataPrefix=m21-xml%2Foai&from=2016-01-01
http://oai.swissbib.ch/oai/DB=2.1?verb=ListIdentifiers&metadataPrefix=m21-xml%2Foai&from=2016-01-01&set=F
......
......@@ -34,5 +34,7 @@ object Dependencies {
lazy val elasticsearch = "org.elasticsearch.client" % "elasticsearch-rest-high-level-client" % "7.7.0"
lazy val json2xml = "de.odysseus.staxon" % "staxon" % "1.3"
}
Supports Markdown
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment