Unverified Commit d6dc65c0 authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

provide means to only download oembed without thumbnail

parent d9c626f1
Pipeline #23970 passed with stages
in 4 minutes and 8 seconds
......@@ -20,7 +20,6 @@ package org.memobase
import ch.memobase.reporting.Report
import com.beust.klaxon.Klaxon
import java.time.LocalDateTime
data class IntermediateReport(
val id: String,
......
......@@ -26,6 +26,10 @@ import ch.memobase.settings.HeaderExtractionTransformSupplier
import ch.memobase.settings.HeaderMetadata
import ch.memobase.settings.SettingsLoader
import ch.memobase.sftp.SftpClient
import java.io.StringReader
import java.io.StringWriter
import java.net.MalformedURLException
import java.net.URL
import org.apache.jena.rdf.model.Model
import org.apache.jena.rdf.model.ModelFactory
import org.apache.jena.rdf.model.Resource
......@@ -36,10 +40,6 @@ import org.apache.kafka.streams.kstream.KStream
import org.apache.kafka.streams.kstream.Predicate
import org.apache.logging.log4j.LogManager
import org.memobase.reports.ReportMessages
import java.io.StringReader
import java.io.StringWriter
import java.net.MalformedURLException
import java.net.URL
class KafkaTopology(private val settings: SettingsLoader) {
private val appSettings = settings.appSettings
......@@ -211,63 +211,68 @@ class KafkaTopology(private val settings: SettingsLoader) {
}
private fun fetchThumbnailForYoutubeOrVimeoFile(value: Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport>): Triple<Pair<Model, HeaderMetadata>, List<Resource>, IntermediateReport> {
if (noThumbnailAttached(value.second)) {
val recordResource = getRecordResource(value.second)
val digitalObjectResource = getDigitalObjectResource(value.second)
if (recordResource != null && digitalObjectResource != null) {
val locator = digitalObjectResource.getProperty(EBUCORE.locator).string
when {
isNoValidUrl(locator) -> {
log.warn("No valid locator url found for ${value.third.id}")
return updateRecord(value, ReportStatus.warning, digitalObjectMessage = "no valid locator url")
}
RemoteResourceHandler.isVimeoUrl(locator) -> {
log.info("Trying to download thumbnail file on vimeo for ${value.third.id}")
val thumbnailHandler = this.previewImageHandler.getFromVimeo(locator)
if (thumbnailHandler == null) {
log.warn("Download for ${value.third.id} failed!")
return updateRecord(
value,
ReportStatus.warning,
digitalObjectMessage = "couldn't download oembed metadata from youtube"
)
}
thumbnailHandler
}
RemoteResourceHandler.isYoutubeUrl(locator) -> {
log.info("Trying to download thumbnail file on youtube for ${value.third.id}")
val thumbnailHandler = this.previewImageHandler.getFromYoutube(locator)
if (thumbnailHandler == null) {
log.warn("Download for ${value.third.id} failed!")
return updateRecord(
value,
ReportStatus.warning,
digitalObjectMessage = "couldn't download oembed metadata from vimeo"
)
}
thumbnailHandler
}
else -> {
log.debug("Won't fetch thumbnail file for ${value.third.id} because no youtube/vimeo resource")
val downloadThumbnail = noThumbnailAttached(value.second)
val recordResource = getRecordResource(value.second)
val digitalObjectResource = getDigitalObjectResource(value.second)
if (recordResource != null && digitalObjectResource != null) {
val locator = digitalObjectResource.getProperty(EBUCORE.locator).string
when {
isNoValidUrl(locator) -> {
log.warn("No valid locator url found for ${value.third.id}")
return updateRecord(value, ReportStatus.warning, digitalObjectMessage = "no valid locator url")
}
RemoteResourceHandler.isVimeoUrl(locator) -> {
log.info("Trying to download oembed metadata on vimeo for ${value.third.id}")
val thumbnailHandler = this.previewImageHandler.getFromVimeo(locator, downloadThumbnail)
if (thumbnailHandler == null) {
log.warn("Download for ${value.third.id} failed!")
return updateRecord(
value,
ReportStatus.ignored,
thumbnailMessage = "no thumbnails available"
ReportStatus.warning,
digitalObjectMessage = "couldn't download oembed metadata from youtube"
)
}
}.let { h ->
val enrichedValue = addDimensionsToDigitalObject(value, h.first)
val filePath = h.second
return if (filePath != null) {
addLocalThumbnail(enrichedValue, recordResource, digitalObjectResource, filePath)
} else {
log.warn("No thumbnail url available for ${value.third.id}")
updateRecord(
enrichedValue,
thumbnailHandler
}
RemoteResourceHandler.isYoutubeUrl(locator) -> {
log.info("Trying to download oembed metadata on youtube for ${value.third.id}")
val thumbnailHandler = this.previewImageHandler.getFromYoutube(locator, downloadThumbnail)
if (thumbnailHandler == null) {
log.warn("Download for ${value.third.id} failed!")
return updateRecord(
value,
ReportStatus.warning,
thumbnailMessage = "Download of youtube / vimeo thumbnail failed or no thumbnail available."
digitalObjectMessage = "couldn't download oembed metadata from vimeo"
)
}
thumbnailHandler
}
else -> {
log.debug("Won't fetch thumbnail file for ${value.third.id} because no youtube/vimeo resource")
return updateRecord(
value,
ReportStatus.ignored,
thumbnailMessage = "no thumbnails available"
)
}
}.let { h ->
val enrichedValue = addDimensionsToDigitalObject(value, h.first)
val filePath = h.second
return if (filePath != null) {
addLocalThumbnail(enrichedValue, recordResource, digitalObjectResource, filePath)
} else if (!downloadThumbnail) {
updateRecord(
enrichedValue,
value.third.status,
thumbnailMessage = "Download of thumbnail skipped because file already present locally"
)
} else {
log.warn("No thumbnail url available for ${value.third.id}")
updateRecord(
enrichedValue,
ReportStatus.warning,
thumbnailMessage = "Download of youtube / vimeo thumbnail failed or no thumbnail available."
)
}
}
}
......@@ -338,7 +343,7 @@ class KafkaTopology(private val settings: SettingsLoader) {
private fun hasAccessibleDigitalObjectWithoutLocator(res: List<Resource>): Boolean {
return res.any {
it.hasProperty(RICO.type, Constant.digitalObject) && !it.hasProperty(EBUCORE.locator) &&
it.hasProperty(RICO.regulatedBy) && !isFaroResource(res)
it.hasProperty(RICO.regulatedBy) && !isFaroResource(res)
}
}
......@@ -360,8 +365,8 @@ class KafkaTopology(private val settings: SettingsLoader) {
private fun hasNoAccessProperty(res: List<Resource>): Boolean {
return res.any {
it.hasProperty(RICO.type, Constant.digitalObject) &&
!it.hasProperty(EBUCORE.locator) &&
!it.hasProperty(RICO.regulatedBy)
!it.hasProperty(EBUCORE.locator) &&
!it.hasProperty(RICO.regulatedBy)
}
}
......
......@@ -115,13 +115,20 @@ class RemoteResourceHandler(private val sftpClient: SftpClient) {
* Get embedding information and preview image from Youtube
*
* @param videoURL URL of video
* @param downloadThumbnail Fetch thumbnail binary
*
* @return Pair with Oembed object and path to local file
*/
fun getFromYoutube(videoURL: String): Pair<OembedResponse, String?>? {
fun getFromYoutube(videoURL: String, downloadThumbnail: Boolean): Pair<OembedResponse, String?>? {
return getYoutubeOembedObject(videoURL)?.let { obj ->
Pair(obj, obj.thumbnail_url?.let {
getThumbnail(it)
if (downloadThumbnail) {
log.info("Trying to download thumbnail from $it")
getThumbnail(it)
} else {
log.info("Skipping downloading thumbnail from $it")
null
}
})
}
}
......@@ -130,13 +137,20 @@ class RemoteResourceHandler(private val sftpClient: SftpClient) {
* Get embedding information and preview image from Vimeo
*
* @param videoURL URL of video
* @param downloadThumbnail Fetch thumbnail binary
*
* @return Pair with Oembed object and path to local file
*/
fun getFromVimeo(videoURL: String): Pair<OembedResponse, String?>? {
fun getFromVimeo(videoURL: String, downloadThumbnail: Boolean): Pair<OembedResponse, String?>? {
return getVimeoOembedObject(videoURL)?.let { obj ->
Pair(obj, obj.thumbnail_url?.let {
getThumbnail(it)
if (downloadThumbnail) {
log.info("Trying to download thumbnail from $it")
getThumbnail(it)
} else {
log.info("Skipping downloading thumbnail from $it")
null
}
})
}
}
......
......@@ -7,4 +7,4 @@ object UtilityFunctions {
fun transformIllegalFilenameCharacters(input: String): String {
return input.replace("/", "_")
}
}
\ No newline at end of file
}
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment