Due to a scheduled upgrade to version 14.10, GitLab will be unavailabe on Monday 30.05., from 19:00 until 20:00.

Unverified Commit cefc96c9 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

set status ignore if nothing is found

parent 43cde44f
Pipeline #22156 passed with stages
in 3 minutes and 26 seconds
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
package ch.memobase package ch.memobase
import ch.memobase.models.{ProcessedRecord, ProcessingFatal, ProcessingSuccess} import ch.memobase.models.{ProcessedRecord, ProcessingFatal, ProcessingIgnore, ProcessingSuccess}
import org.apache.logging.log4j.scala.Logging import org.apache.logging.log4j.scala.Logging
import java.io.File import java.io.File
...@@ -30,27 +30,31 @@ trait BinaryHandler { ...@@ -30,27 +30,31 @@ trait BinaryHandler {
self: Logging with AppSettings => self: Logging with AppSettings =>
def deleteBinaries(record: ProcessedRecord): ProcessedRecord = { def deleteBinaries(record: ProcessedRecord): ProcessedRecord = {
val result = record.binaries if (record.binaries.nonEmpty) {
.map(_.replaceFirst("file://", "")) val result = record.binaries
.map(fileName => (new File(fileName), fileName)) .map(_.replaceFirst("file://", ""))
.map(file => (file._1.delete, file._2)) .map(fileName => (new File(fileName), fileName))
.foldLeft[List[String]](List())((errors, x) => .map(file => (file._1.delete, file._2))
if (!x._1) { .foldLeft[List[String]](List())((errors, x) =>
logger.warn(s"Deletion of file ${x._2} failed!") if (!x._1) {
errors :+ s"${x._2} deletion failed" logger.warn(s"Deletion of file ${x._2} failed!")
} else { errors :+ s"${x._2} deletion failed"
errors } else {
} errors
) }
if (result.isEmpty) { )
logger.debug(s"Deletion of binaries for record ${record.recordId} successful") if (result.isEmpty) {
record.setBinariesStatus( logger.debug(s"Deletion of binaries for record ${record.recordId} successful")
ProcessingSuccess, record.setBinariesStatus(
"Binaries deletion successful" ProcessingSuccess,
) "Binaries deletion successful"
)
} else {
logger.warn(s"Deletion of binaries for record ${record.recordId} failed")
record.setBinariesStatus(ProcessingFatal, result.mkString("; "))
}
} else { } else {
logger.warn(s"Deletion of binaries for record ${record.recordId} failed") record.setBinariesStatus(ProcessingIgnore, "No attached local binaries")
record.setBinariesStatus(ProcessingFatal, result.mkString("; "))
} }
} }
} }
...@@ -18,7 +18,7 @@ ...@@ -18,7 +18,7 @@
package ch.memobase package ch.memobase
import ch.memobase.models.{ProcessedRecord, ProcessingFatal, ProcessingIgnore, ProcessingSuccess} import ch.memobase.models._
import org.apache.logging.log4j.scala.Logging import org.apache.logging.log4j.scala.Logging
import java.sql.{Connection, DriverManager} import java.sql.{Connection, DriverManager}
...@@ -27,13 +27,16 @@ import scala.util.{Failure, Success, Try} ...@@ -27,13 +27,16 @@ import scala.util.{Failure, Success, Try}
trait MariadbClient { trait MariadbClient {
self: Logging with AppSettings => self: Logging with AppSettings =>
type DeletionResult = private type DeletionResult =
Try[(Option[String], Boolean, Boolean, Boolean, Boolean)] Try[(Option[String], Boolean, Boolean, Boolean, Boolean)]
private val noRows = Success((None, false, false, false, false))
private var connection = getConnection private var connection = getConnection
private def getConnection: Connection = { private def getConnection: Connection = {
val c = DriverManager.getConnection(mariaDBHost, mariaDBUser, mariaDBPassword) val c =
DriverManager.getConnection(mariaDBHost, mariaDBUser, mariaDBPassword)
c.setAutoCommit(false) c.setAutoCommit(false)
c c
} }
...@@ -111,9 +114,11 @@ trait MariadbClient { ...@@ -111,9 +114,11 @@ trait MariadbClient {
ProcessingFatal, ProcessingFatal,
s"Snippet entries deletion failed: ${e.getMessage}" s"Snippet entries deletion failed: ${e.getMessage}"
) )
case (`noRows`, `noRows`) =>
record.setThumbnailStatus(ProcessingIgnore, "No thumbnails found")
case ( case (
Success((uri, entities, metadata, manifestV2, manifestV3)), Success((uri, entities, metadata, manifestV2, manifestV3)),
Success((None, false, false, false, false)) `noRows`
) => ) =>
(if (uri.isDefined && isLocalFile(uri.get)) { (if (uri.isDefined && isLocalFile(uri.get)) {
record.addBinary(uri.get) record.addBinary(uri.get)
...@@ -131,7 +136,7 @@ trait MariadbClient { ...@@ -131,7 +136,7 @@ trait MariadbClient {
) )
) )
case ( case (
Success((None, false, false, false, false)), `noRows`,
Success((uri, entities, metadata, manifestV2, manifestV3)) Success((uri, entities, metadata, manifestV2, manifestV3))
) => ) =>
(if (uri.isDefined && isLocalFile(uri.get)) { (if (uri.isDefined && isLocalFile(uri.get)) {
...@@ -150,8 +155,10 @@ trait MariadbClient { ...@@ -150,8 +155,10 @@ trait MariadbClient {
) )
) )
case (_, _) => case (_, _) =>
record.setThumbnailStatus(ProcessingIgnore, "No thumbnails found") record.setThumbnailStatus(
// No poster, no snippet ProcessingWarning,
"Snippets and posters found. This should not happen!"
)
} }
} }
...@@ -182,8 +189,8 @@ trait MariadbClient { ...@@ -182,8 +189,8 @@ trait MariadbClient {
} }
def checkDBConnection(timeout: Int): Unit = { def checkDBConnection(timeout: Int): Unit = {
if (connection == null || !connection.isValid(timeout)) { if (connection == null || !connection.isValid(timeout)) {
logger.info("DB connection is non-existent or invalid.") logger.info("DB connection is non-existent or invalid.")
if (!connection.isClosed) { if (!connection.isClosed) {
connection.close() connection.close()
} }
......
...@@ -18,17 +18,12 @@ ...@@ -18,17 +18,12 @@
package ch.memobase package ch.memobase
import ch.memobase.models.{ProcessedRecord, ProcessingFatal, ProcessingSuccess} import ch.memobase.models.{ProcessedRecord, ProcessingFatal, ProcessingIgnore, ProcessingSuccess}
import com.sksamuel.elastic4s import com.sksamuel.elastic4s
import com.sksamuel.elastic4s.ElasticApi.deleteById import com.sksamuel.elastic4s.ElasticApi.deleteById
import com.sksamuel.elastic4s.ElasticDsl._ import com.sksamuel.elastic4s.ElasticDsl._
import com.sksamuel.elastic4s.http.JavaClient import com.sksamuel.elastic4s.http.JavaClient
import com.sksamuel.elastic4s.{ import com.sksamuel.elastic4s.{ElasticClient, ElasticProperties, RequestFailure, RequestSuccess}
ElasticClient,
ElasticProperties,
RequestFailure,
RequestSuccess
}
import org.apache.logging.log4j.scala.Logging import org.apache.logging.log4j.scala.Logging
import scala.concurrent.ExecutionContext.Implicits.global import scala.concurrent.ExecutionContext.Implicits.global
...@@ -48,12 +43,19 @@ trait MemobaseElasticClient { ...@@ -48,12 +43,19 @@ trait MemobaseElasticClient {
deleteById(esIndex, record.recordId) deleteById(esIndex, record.recordId)
} }
.map { .map {
case success: RequestSuccess[_] => case success: RequestSuccess[_] if success.status < 400 =>
logger.debug(s"Deletion of record ${record.recordId} successful") logger.debug(s"Deletion of record ${record.recordId} successful")
record.setRecordStatus( record.setRecordStatus(
ProcessingSuccess, ProcessingSuccess,
"Record deletion in Elasticsearch successful" "Record deletion in Elasticsearch successful"
) )
case success: RequestSuccess[_] =>
logger.debug(s"Deletion of record ${record.recordId} failed since record does not exists")
record.setRecordStatus(
ProcessingIgnore,
"Record does not exist in Elasticearch"
)
case failure: RequestFailure => case failure: RequestFailure =>
logger.warn(s"Deletion of record ${record.recordId} failed: ${failure.error.reason}") logger.warn(s"Deletion of record ${record.recordId} failed: ${failure.error.reason}")
record.setRecordStatus( record.setRecordStatus(
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment