Unverified Commit d9adb7fc authored by Sebastian Schüpbach's avatar Sebastian Schüpbach
Browse files

allow for custom mappings


Signed-off-by: Sebastian Schüpbach's avatarSebastian Schüpbach <sebastian.schuepbach@unibas.ch>
parent cb18debf
......@@ -68,3 +68,5 @@ The server assumes a file tree according to this scheme:
Importantly, the `media` and `thumbnails` directory have to be direct subfolders of the recordSet directories and must directly contain the media files.
When generating the IDs for the files, whitespace in filenames are replaced with underscored (`_`).
If a file name does not match with the id, you can define a custom mapping. See example config for details.
......@@ -32,6 +32,7 @@ use hyper::service::make_service_fn;
use hyper::{Error, Server};
use log::info;
use serde::Deserialize;
use std::collections::HashMap;
use std::env;
use std::fs::File;
use std::io::prelude::*;
......@@ -40,6 +41,9 @@ use std::str::FromStr;
use std::sync::{Arc, Mutex};
use tokio::time::Duration;
type CustomCollectionMappings = HashMap<String, HashMap<String, String>>;
type CustomMappings = HashMap<String, CustomCollectionMappings>;
#[tokio::main]
async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
env_logger::init();
......@@ -57,8 +61,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error + Send + Sync>> {
"Reading in media files starting with path {}",
&config.base_path
);
let mut media_cache = MediaFileCache::new(config.base_path.clone());
media_cache.refresh().context("Refreshing cache failed!")?;
let media_cache = MediaFileCache::new(config.base_path.clone(), config.mappings.clone());
info!(
"Done reading in media files. Found {} metadata files, {} media files and {} thumbnails",
media_cache.metadata_file_size(),
......@@ -113,4 +116,6 @@ pub struct Config {
pub cache_timeout: Option<u64>,
/// Timeout (in seconds) after which a collection cache is updated if a query for a certain ID failed
pub partial_cache_timeout: Option<u64>,
/// Mappings for special file names
pub mappings: Option<CustomMappings>,
}
......@@ -16,6 +16,7 @@
* along with this program. If not, see <https://www.gnu.org/licenses/>.
*/
use crate::{CustomCollectionMappings, CustomMappings};
use anyhow::anyhow;
use anyhow::{Context, Result};
use log::{debug, warn};
......@@ -31,8 +32,12 @@ pub struct Collection {
dissemination_copies: HashMap<String, String>,
/// Index of thumbnail files in collection
thumbnails: HashMap<String, String>,
/// Number of dissemination copies in collection
num_dissemination_copies: usize,
/// Number of thumbnails in collection
num_thumbnails: usize,
/// Number of metadata files in collection
metadata_file_count: usize,
num_metadata_files: usize,
/// File path to collection
base_path: String,
/// Creation instance of collection. Set to [`std::time::Instant::now()`] when collection has been updated.
......@@ -41,7 +46,7 @@ pub struct Collection {
impl Collection {
/// Creates a new collection
pub fn new(base_path: String) -> Self {
pub fn new(base_path: String, custom_mappings: Option<CustomCollectionMappings>) -> Self {
let base = Path::new(&base_path);
let parse = |r: &str| {
let path = base.join(r);
......@@ -66,10 +71,48 @@ impl Collection {
HashMap::new()
}
};
let mut dissemination_copies = parse("media");
let num_dissemination_copies = dissemination_copies.len();
let mut thumbnails = parse("thumbnails");
let num_thumbnails = thumbnails.len();
if let Some(ids) = custom_mappings {
if let Some(media_ids) = ids.get("media") {
let media_ids_cloned = media_ids
.to_owned()
.into_iter()
.map(|v| {
(
v.0,
base.join("media").join(v.1).to_str().unwrap().to_string(),
)
})
.collect::<HashMap<String, String>>();
dissemination_copies.extend(media_ids_cloned);
}
if let Some(thumbnail_ids) = ids.get("thumbnails") {
let thumbnail_ids_cloned = thumbnail_ids
.to_owned()
.into_iter()
.map(|v| {
(
v.0,
base.join("thumbnails")
.join(v.1)
.to_str()
.unwrap()
.to_string(),
)
})
.collect::<HashMap<String, String>>();
thumbnails.extend(thumbnail_ids_cloned);
}
}
Collection {
dissemination_copies: parse("media"),
thumbnails: parse("thumbnails"),
metadata_file_count: Collection::count_metadata_files(base).unwrap_or_else(|_| {
dissemination_copies,
thumbnails,
num_dissemination_copies,
num_thumbnails,
num_metadata_files: Collection::count_metadata_files(base).unwrap_or_else(|_| {
warn!("No metadata files found in {}", &base_path);
0
}),
......@@ -167,17 +210,17 @@ impl Collection {
/// Returns index size for dissemination copies (i.e. the total of cached dissemination copies)
pub fn dissemination_copies_size(&self) -> usize {
self.dissemination_copies.len()
self.num_dissemination_copies
}
/// Returns index size for thumbnails (i.e. the total of cached thumbnails)
pub fn thumbnails_size(&self) -> usize {
self.thumbnails.len()
self.num_thumbnails
}
/// Retuns number of metadata files in collection
pub fn metadata_files(&self) -> usize {
self.metadata_file_count
self.num_metadata_files
}
}
......@@ -186,6 +229,8 @@ impl Collection {
pub struct MediaFileCache {
/// Collection index
collections: HashMap<String, Collection>,
/// Mappings for special file names
custom_mappings: Option<CustomMappings>,
/// Path to root directory
base_path: String,
/// Creation instance of cache. Set to [`std::time::Instant::now()`] when cache has been updated
......@@ -194,23 +239,28 @@ pub struct MediaFileCache {
impl MediaFileCache {
/// Creates a new instance
pub fn new(base_path: String) -> Self {
let collections =
if let Ok(index) = MediaFileCache::scan_collection_folders(Path::new(&base_path)) {
index
} else {
warn!("Collection indexing failed!");
HashMap::new()
};
pub fn new(base_path: String, custom_mappings: Option<CustomMappings>) -> Self {
let collections = if let Ok(index) =
MediaFileCache::scan_collection_folders(Path::new(&base_path), &custom_mappings)
{
index
} else {
warn!("Collection indexing failed!");
HashMap::new()
};
MediaFileCache {
collections,
custom_mappings,
base_path,
created_on: Instant::now(),
}
}
/// Scans the root path for collections and returns an index of collection IDs and their files
fn scan_collection_folders(dir: &Path) -> Result<HashMap<String, Collection>> {
fn scan_collection_folders(
dir: &Path,
custom_mappings: &Option<CustomMappings>,
) -> Result<HashMap<String, Collection>> {
let mut file_index: HashMap<String, Collection> = HashMap::new();
for entry in fs::read_dir(&dir).context("Can't read dir")? {
let entry = entry?;
......@@ -223,7 +273,16 @@ impl MediaFileCache {
.to_str()
.context("Can't convert OsStr to str")?
.to_owned();
file_index.insert(dir_name, Collection::new(path_as_string));
let custom_collection_mappings =
if let Some(m) = custom_mappings.as_ref().and_then(|m| m.get(&dir_name)) {
Some(m.clone())
} else {
None
};
file_index.insert(
dir_name,
Collection::new(path_as_string, custom_collection_mappings),
);
}
}
Ok(file_index)
......@@ -312,8 +371,11 @@ impl MediaFileCache {
/// Refreshes the entire cache
pub fn refresh(&mut self) -> Result<()> {
self.collections = MediaFileCache::scan_collection_folders(Path::new(&self.base_path))
.context("Cache refreshing failed")?;
self.collections = MediaFileCache::scan_collection_folders(
Path::new(&self.base_path),
&self.custom_mappings,
)
.context("Cache refreshing failed")?;
self.created_on = Instant::now();
Ok(())
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment