From ecfbf2b474fb527af24d45382e3260b264ed20ba Mon Sep 17 00:00:00 2001 From: hamidreza kalbasi Date: Fri, 4 Jun 2021 08:06:12 +0430 Subject: [PATCH] put media in filesystem --- src/client_server/media.rs | 14 +++++---- src/database/globals.rs | 21 ++++++++++---- src/database/media.rs | 58 ++++++++++++++++++++++++++++---------- src/error.rs | 5 ++++ 4 files changed, 73 insertions(+), 25 deletions(-) diff --git a/src/client_server/media.rs b/src/client_server/media.rs index f9350e0..f59c0a8 100644 --- a/src/client_server/media.rs +++ b/src/client_server/media.rs @@ -38,10 +38,11 @@ pub async fn create_content_route( ); db.media.create( mxc.clone(), + &db.globals, &body.filename.as_deref(), &body.content_type.as_deref(), &body.file, - )?; + ).await?; db.flush().await?; @@ -67,7 +68,7 @@ pub async fn get_content_route( filename, content_type, file, - }) = db.media.get(&mxc)? + }) = db.media.get(&db.globals, &mxc).await? { Ok(get_content::Response { file, @@ -91,10 +92,11 @@ pub async fn get_content_route( db.media.create( mxc, + &db.globals, &get_content_response.content_disposition.as_deref(), &get_content_response.content_type.as_deref(), &get_content_response.file, - )?; + ).await?; Ok(get_content_response.into()) } else { @@ -117,13 +119,14 @@ pub async fn get_content_thumbnail_route( content_type, file, .. }) = db.media.get_thumbnail( mxc.clone(), + &db.globals, body.width .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, body.height .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, - )? { + ).await? { Ok(get_content_thumbnail::Response { file, content_type }.into()) } else if &*body.server_name != db.globals.server_name() && body.allow_remote { let get_thumbnail_response = db @@ -144,12 +147,13 @@ pub async fn get_content_thumbnail_route( db.media.upload_thumbnail( mxc, + &db.globals, &None, &get_thumbnail_response.content_type, body.width.try_into().expect("all UInts are valid u32s"), body.height.try_into().expect("all UInts are valid u32s"), &get_thumbnail_response.file, - )?; + ).await?; Ok(get_thumbnail_response.into()) } else { diff --git a/src/database/globals.rs b/src/database/globals.rs index 5d91d37..311ff25 100644 --- a/src/database/globals.rs +++ b/src/database/globals.rs @@ -5,11 +5,7 @@ use ruma::{ EventId, MilliSecondsSinceUnixEpoch, ServerName, ServerSigningKeyId, }; use rustls::{ServerCertVerifier, WebPKIVerifier}; -use std::{ - collections::{BTreeMap, HashMap}, - sync::{Arc, RwLock}, - time::{Duration, Instant}, -}; +use std::{collections::{BTreeMap, HashMap}, path::{PathBuf}, sync::{Arc, RwLock}, time::{Duration, Instant}}; use tokio::sync::Semaphore; use trust_dns_resolver::TokioAsyncResolver; @@ -264,4 +260,19 @@ impl Globals { self.globals.insert("version", &new_version.to_be_bytes())?; Ok(()) } + + pub fn get_media_folder(&self) -> PathBuf { + let mut r = PathBuf::new(); + r.push(self.config.database_path.clone()); + r.push("media"); + r + } + + pub fn get_media_file(&self, key: &Vec) -> PathBuf { + let mut r = PathBuf::new(); + r.push(self.config.database_path.clone()); + r.push("media"); + r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD)); + r + } } diff --git a/src/database/media.rs b/src/database/media.rs index 37fcb74..6a86c04 100644 --- a/src/database/media.rs +++ b/src/database/media.rs @@ -1,7 +1,9 @@ use image::{imageops::FilterType, GenericImageView}; +use crate::database::globals::Globals; use crate::{utils, Error, Result}; -use std::mem; +use std::{mem}; +use tokio::{fs::{self, File}, io::AsyncWriteExt, io::AsyncReadExt}; pub struct FileMeta { pub filename: Option, @@ -15,10 +17,11 @@ pub struct Media { } impl Media { - /// Uploads or replaces a file. - pub fn create( + /// Uploads a file. + pub async fn create( &self, mxc: String, + globals: &Globals, filename: &Option<&str>, content_type: &Option<&str>, file: &[u8], @@ -37,15 +40,20 @@ impl Media { .unwrap_or_default(), ); - self.mediaid_file.insert(key, file)?; + let path = globals.get_media_file(&key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(file).await?; + self.mediaid_file.insert(key, vec![])?; Ok(()) } /// Uploads or replaces a file thumbnail. - pub fn upload_thumbnail( + pub async fn upload_thumbnail( &self, mxc: String, + globals: &Globals, filename: &Option, content_type: &Option, width: u32, @@ -66,13 +74,18 @@ impl Media { .unwrap_or_default(), ); - self.mediaid_file.insert(key, file)?; + let path = globals.get_media_file(&key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(file).await?; + + self.mediaid_file.insert(key, vec![])?; Ok(()) } /// Downloads a file. - pub fn get(&self, mxc: &str) -> Result> { + pub async fn get(&self, globals: &Globals, mxc: &str) -> Result> { let mut prefix = mxc.as_bytes().to_vec(); prefix.push(0xff); prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail @@ -80,7 +93,10 @@ impl Media { prefix.push(0xff); if let Some(r) = self.mediaid_file.scan_prefix(&prefix).next() { - let (key, file) = r?; + let (key, _file) = r?; + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -107,7 +123,7 @@ impl Media { Ok(Some(FileMeta { filename, content_type, - file: file.to_vec(), + file, })) } else { Ok(None) @@ -137,7 +153,7 @@ impl Media { /// - Server creates the thumbnail and sends it to the user /// /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards. - pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result> { + pub async fn get_thumbnail(&self, mxc: String, globals: &Globals, width: u32, height: u32) -> Result> { let (width, height, crop) = self .thumbnail_properties(width, height) .unwrap_or((0, 0, false)); // 0, 0 because that's the original file @@ -157,7 +173,10 @@ impl Media { if let Some(r) = self.mediaid_file.scan_prefix(&thumbnail_prefix).next() { // Using saved thumbnail - let (key, file) = r?; + let (key, _file) = r?; + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -190,7 +209,11 @@ impl Media { } else if let Some(r) = self.mediaid_file.scan_prefix(&original_prefix).next() { // Generate a thumbnail - let (key, file) = r?; + let (key, _file) = r?; + let path = globals.get_media_file(&key.to_vec()); + let mut file = vec![]; + File::open(path).await?.read_to_end(&mut file).await?; + let mut parts = key.rsplit(|&b| b == 0xff); let content_type = parts @@ -283,19 +306,24 @@ impl Media { widthheight, ); - self.mediaid_file.insert(thumbnail_key, &*thumbnail_bytes)?; + let path = globals.get_media_file(&thumbnail_key); + fs::create_dir_all(path.parent().unwrap()).await?; + let mut f = File::create(path).await?; + f.write_all(&thumbnail_bytes).await?; + + self.mediaid_file.insert(thumbnail_key, vec![])?; Ok(Some(FileMeta { filename, content_type, - file: thumbnail_bytes.to_vec(), + file: thumbnail_bytes.to_vec() })) } else { // Couldn't parse file to generate thumbnail, send original Ok(Some(FileMeta { filename, content_type, - file: file.to_vec(), + file: file.to_vec() })) } } else { diff --git a/src/error.rs b/src/error.rs index e2664e2..e36f863 100644 --- a/src/error.rs +++ b/src/error.rs @@ -40,6 +40,11 @@ pub enum Error { }, #[error("{0}")] FederationError(Box, RumaError), + #[error("Could not do this io: {source}")] + IoError { + #[from] + source: std::io::Error, + }, #[error("{0}")] BadServerResponse(&'static str), #[error("{0}")]