Commit bc48006a authored by Jonas Waeber's avatar Jonas Waeber
Browse files

Add field numberOfDocumentsPublished.

parent f3537273
Pipeline #26096 passed with stages
in 5 minutes and 23 seconds
......@@ -113,7 +113,10 @@ class InstitutionSearchDocBuilder(
postalCodes = postalCodes,
numberOfRecordSets = recordSetIds.count(),
numberOfDocuments = recordSetIds
.map { value -> elasticSearchWrapper.countNumberOfDocuments(value) }
.map { value -> elasticSearchWrapper.countTotalNumberOfDocuments(value) }
.sum(),
numberOfDocumentsPublished = recordSetIds
.map { value -> elasticSearchWrapper.countNumberOfDocumentsPublished(value) }
.sum(),
teaserColor = institution[MB.institutionTeaserColor.localName].let {
if (it == null) {
......
......@@ -179,7 +179,8 @@ class RecordSetSearchDocBuilder(private val elasticSearchWrapper: ElasticSearchW
name = name,
documentType = elasticSearchWrapper.getDocumentTypesFromRecords(id, Constants.QueryFields.recordSetFacet),
keyVisualLink = recordSet[Constants.wikidataImage].let { if (it != null) it as String else "NoKeyVisualLinkDefined" },
numberOfDocuments = elasticSearchWrapper.countNumberOfDocuments(id),
numberOfDocuments = elasticSearchWrapper.countTotalNumberOfDocuments(id),
numberOfDocumentsPublished = elasticSearchWrapper.countNumberOfDocumentsPublished(id),
lastUpdatedDate = Date.now,
languageOfMetadata = metadataLanguages.map {
FacetContainer(
......
......@@ -29,6 +29,8 @@ import org.elasticsearch.client.RequestOptions
import org.elasticsearch.client.RestHighLevelClient
import org.elasticsearch.client.core.CountRequest
import org.elasticsearch.common.unit.TimeValue
import org.elasticsearch.index.query.QueryBuilder
import org.elasticsearch.index.query.QueryBuilders.boolQuery
import org.elasticsearch.index.query.QueryBuilders.termQuery
import org.elasticsearch.search.Scroll
import org.elasticsearch.search.builder.SearchSourceBuilder
......@@ -50,6 +52,25 @@ class ElasticSearchWrapper(
private val log = LogManager.getLogger("ElasticSearchWrapper")
private val klaxon = Klaxon()
fun countTotalNumberOfDocuments(recordSetIdentifier: String): Int {
return countNumberOfDocuments(
recordSetIdentifier,
termQuery(
"recordSet.facet", recordSetIdentifier
)
)
}
fun countNumberOfDocumentsPublished(recordSetIdentifier: String): Int {
return countNumberOfDocuments(
recordSetIdentifier,
boolQuery()
.must(termQuery("recordSet.facet", recordSetIdentifier))
.must(termQuery("published", true))
)
}
/**
* Counts the number of documents attached to a specific record set.
*
......@@ -57,14 +78,10 @@ class ElasticSearchWrapper(
*
* @return Number of Documents
*/
fun countNumberOfDocuments(recordSetIdentifier: String): Int {
private fun countNumberOfDocuments(recordSetIdentifier: String, query: QueryBuilder): Int {
log.info("Counting documents for record set $recordSetIdentifier from index $documentsIndex.")
val request = CountRequest(documentsIndex)
request.query(
termQuery(
"recordSet.facet", recordSetIdentifier
)
)
request.query(query)
return try {
val response = client.count(
request, RequestOptions.DEFAULT
......
......@@ -46,6 +46,7 @@ data class InstitutionSearchDoc(
val keyVisualLink: String,
val numberOfRecordSets: Int,
val numberOfDocuments: Int,
val numberOfDocumentsPublished: Int,
val teaserColor: String,
val teaserColorComputed: String
) : Schema(institutionId) {
......@@ -64,7 +65,8 @@ data class InstitutionSearchDoc(
Date.now,
"",
0,
0,
100,
90,
"",
""
)
......
......@@ -31,6 +31,7 @@ data class RecordSetSearchDoc(
// Display
val keyVisualLink: String,
val numberOfDocuments: Int,
val numberOfDocumentsPublished: Int,
val teaserTest: LanguageContainer,
// Facets
val documentType: List<FacetContainer>,
......@@ -95,6 +96,7 @@ data class RecordSetSearchDoc(
lastUpdatedDate = "2020-11-20T10:29:01.128",
keyVisualLink = "https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2020-10/1.jpg?itok=5ncVBnVQ",
numberOfDocuments = 100,
numberOfDocumentsPublished = 90,
scopeAndContent = LanguageContainer.placeholder(lorem),
accessMemobase = LanguageContainer.placeholder(lorem),
context = LanguageContainer.placeholder(lorem),
......
......@@ -62,7 +62,8 @@ class IntegrationTest {
fun `integration tests`(params: TestParam) {
val settings = App.createSettings("kafkaTest1.yml")
every { elasticSearchWrapperMocked.countNumberOfDocuments("") } returns 123
every { elasticSearchWrapperMocked.countTotalNumberOfDocuments("") } returns 100
every { elasticSearchWrapperMocked.countNumberOfDocumentsPublished("") } returns 90
every { elasticSearchWrapperMocked.getDocumentTypesFromRecords("", "") } returns listOf(
FacetContainer(
LanguageContainer(listOf("Foto"), listOf("Foto"), listOf("Foto"), emptyList()), "Foto", emptyList()
......
......@@ -72,7 +72,8 @@ class TestInstitutionSearchDoc {
fun `integration test institution`() {
val settings = App.createSettings("kafkaTest1.yml")
every { elasticSearchWrapperMocked.countNumberOfDocuments("testComplete") } returns 123
every { elasticSearchWrapperMocked.countTotalNumberOfDocuments("testComplete") } returns 100
every { elasticSearchWrapperMocked.countNumberOfDocumentsPublished("testComplete") } returns 90
every {
elasticSearchWrapperMocked.getDocumentTypesFromRecords(
"testComplete",
......
......@@ -54,7 +54,8 @@ class TestRecordSetSearchDoc {
val data = readFile("completeExample.json")
val wrapper = mockk<ElasticSearchWrapper>()
every { wrapper.countNumberOfDocuments("testComplete") } returns 102
every { wrapper.countTotalNumberOfDocuments("testComplete") } returns 100
every { wrapper.countNumberOfDocumentsPublished("testComplete") } returns 90
every { wrapper.getDocumentTypesFromRecords("testComplete", "recordSet.facet") } returns listOf(
FacetContainer(
LanguageContainer(listOf("Fotographie"), listOf("Photographie"), listOf("Fotografia"), emptyList()),
......@@ -106,7 +107,8 @@ class TestRecordSetSearchDoc {
fun `integration test record sets`() {
val settings = App.createSettings("kafkaTest1.yml")
every { elasticSearchWrapperMocked.countNumberOfDocuments("") } returns 123
every { elasticSearchWrapperMocked.countTotalNumberOfDocuments("") } returns 100
every { elasticSearchWrapperMocked.countNumberOfDocumentsPublished("") } returns 90
every { elasticSearchWrapperMocked.getDocumentTypesFromRecords("", "") } returns listOf(
FacetContainer(
LanguageContainer(listOf("Foto"), listOf("Foto"), listOf("Foto"), emptyList()), "Foto", emptyList()
......
{"published":false,"name":{"de":["Complete Institution (DE)"],"fr":["Complete Institution (FR)"],"it":["Complete Institution (IT)"],"un":[]},"description":{"de":["<p>Beschreibung (DE)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"],"fr":["<p>Beschreibung (FR)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"],"it":["<p>Beschreibung (IT)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"],"un":[]},"city":[{"de":[],"fr":[],"it":[],"un":["City"]}],"address":["Street Address"],"postalCodes":["1000"],"canton":[{"name":{"de":["Aargau"],"fr":["Argovie"],"it":["Argovia"],"un":[]}}],"type":[{"name":{"de":["Rundfunkveranstalter"],"fr":["Radio- et télédiffuseur"],"it":["Emittente"],"un":[]},"filter":"Q15265344"},{"name":{"de":["Veranstalter"],"fr":["Organisateur"],"it":["Organizzatore"],"un":[]},"filter":"Q2029941"},{"name":{"de":["Privatwirtschaft"],"fr":["Secteur privé"],"it":["Settore privato"],"un":[]},"filter":"Q591763"}],"documentType":[{"name":{"de":["Foto"],"fr":["Foto"],"it":["Foto"],"un":[]},"filter":"Foto"}],"lastUpdatedDate":"2020","keyVisualLink":"https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2021-02/vitrine1_hero.jpg?itok=S-b5nq1p","numberOfRecordSets":1,"numberOfDocuments":123,"id":"completeInstitution"}
\ No newline at end of file
{"published":false,"name":{"de":["Complete Institution (DE)"],"fr":["Complete Institution (FR)"],"it":["Complete Institution (IT)"],"un":[]},"description":{"de":["<p>Beschreibung (DE)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"],"fr":["<p>Beschreibung (FR)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"],"it":["<p>Beschreibung (IT)</p>\r\n\r\n<p>Lorem ipsum dolor sit amet, consectetur adipiscing elit, sed do eiusmod tempor incididunt ut labore et dolore magna aliqua. Ut enim ad minim veniam, quis nostrud exercitation ullamco laboris nisi ut aliquip ex ea commodo consequat. Duis aute irure dolor in reprehenderit in voluptate velit esse cillum dolore eu fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident, sunt in culpa qui officia deserunt mollit anim id est laborum.</p>"],"un":[]},"city":[{"de":[],"fr":[],"it":[],"un":["City"]}],"address":["Street Address"],"postalCodes":["1000"],"canton":[{"name":{"de":["Aargau"],"fr":["Argovie"],"it":["Argovia"],"un":[]}}],"type":[{"name":{"de":["Rundfunkveranstalter"],"fr":["Radio- et télédiffuseur"],"it":["Emittente"],"un":[]},"filter":"Q15265344"},{"name":{"de":["Veranstalter"],"fr":["Organisateur"],"it":["Organizzatore"],"un":[]},"filter":"Q2029941"},{"name":{"de":["Privatwirtschaft"],"fr":["Secteur privé"],"it":["Settore privato"],"un":[]},"filter":"Q591763"}],"documentType":[{"name":{"de":["Foto"],"fr":["Foto"],"it":["Foto"],"un":[]},"filter":"Foto"}],"lastUpdatedDate":"2020","keyVisualLink":"https://mb-wf1.memobase.unibas.ch/sites/default/files/styles/teaser/public/2021-02/vitrine1_hero.jpg?itok=S-b5nq1p","numberOfRecordSets":1,"numberOfDocuments":100,"numberOfDocumentsPublished":90,"id":"completeInstitution"}
\ No newline at end of file
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment