Merge branch 'master' into 'master'

put media in filesystem

See merge request famedly/conduit!95

src/client_server/media.rs

@@ -36,16 +36,19 @@ pub async fn create_content_route(
         db.globals.server_name(),
         utils::random_string(MXC_LENGTH)
     );
-    db.media.create(
-        mxc.clone(),
-        &body
-            .filename
-            .as_ref()
-            .map(|filename| "inline; filename=".to_owned() + filename)
-            .as_deref(),
-        &body.content_type.as_deref(),
-        &body.file,
-    )?;
+    db.media
+        .create(
+            mxc.clone(),
+            &db.globals,
+            &body
+                .filename
+                .as_ref()
+                .map(|filename| "inline; filename=".to_owned() + filename)
+                .as_deref(),
+            &body.content_type.as_deref(),
+            &body.file,
+        )
+        .await?;
 
     db.flush().await?;
 
@@ -71,7 +74,7 @@ pub async fn get_content_route(
         content_disposition,
         content_type,
         file,
-    }) = db.media.get(&mxc)?
+    }) = db.media.get(&db.globals, &mxc).await?
     {
         Ok(get_content::Response {
             file,
@@ -93,12 +96,15 @@ pub async fn get_content_route(
             )
             .await?;
 
-        db.media.create(
-            mxc,
-            &get_content_response.content_disposition.as_deref(),
-            &get_content_response.content_type.as_deref(),
-            &get_content_response.file,
-        )?;
+        db.media
+            .create(
+                mxc,
+                &db.globals,
+                &get_content_response.content_disposition.as_deref(),
+                &get_content_response.content_type.as_deref(),
+                &get_content_response.file,
+            )
+            .await?;
 
         Ok(get_content_response.into())
     } else {
@@ -119,15 +125,20 @@ pub async fn get_content_thumbnail_route(
 
     if let Some(FileMeta {
         content_type, file, ..
-    }) = db.media.get_thumbnail(
-        mxc.clone(),
-        body.width
-            .try_into()
-            .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
-        body.height
-            .try_into()
-            .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
-    )? {
+    }) = db
+        .media
+        .get_thumbnail(
+            mxc.clone(),
+            &db.globals,
+            body.width
+                .try_into()
+                .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
+            body.height
+                .try_into()
+                .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
+        )
+        .await?
+    {
         Ok(get_content_thumbnail::Response { file, content_type }.into())
     } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
         let get_thumbnail_response = db
@@ -146,14 +157,17 @@ pub async fn get_content_thumbnail_route(
             )
             .await?;
 
-        db.media.upload_thumbnail(
-            mxc,
-            &None,
-            &get_thumbnail_response.content_type,
-            body.width.try_into().expect("all UInts are valid u32s"),
-            body.height.try_into().expect("all UInts are valid u32s"),
-            &get_thumbnail_response.file,
-        )?;
+        db.media
+            .upload_thumbnail(
+                mxc,
+                &db.globals,
+                &None,
+                &get_thumbnail_response.content_type,
+                body.width.try_into().expect("all UInts are valid u32s"),
+                body.height.try_into().expect("all UInts are valid u32s"),
+                &get_thumbnail_response.file,
+            )
+            .await?;
 
         Ok(get_thumbnail_response.into())
     } else {

src/database.rs

@@ -20,7 +20,8 @@ use ruma::{DeviceId, ServerName, UserId};
 use serde::Deserialize;
 use std::{
     collections::HashMap,
-    fs::remove_dir_all,
+    fs::{self, remove_dir_all},
+    io::Write,
     sync::{Arc, RwLock},
 };
 use tokio::sync::Semaphore;
@@ -253,9 +254,11 @@ impl Database {
 
                 let password = utils::string_from_bytes(&password);
 
-                if password.map_or(false, |password| {
+                let password_not_exists = password.map_or(false, |password| {
                     argon2::verify_encoded(&password, b"").unwrap_or(false)
-                }) {
+                });
+
+                if password_not_exists {
                     db.users.userid_password.insert(userid, b"")?;
                 }
             }
@@ -265,6 +268,21 @@ impl Database {
             info!("Migration: 1 -> 2 finished");
         }
 
+        if db.globals.database_version()? < 3 {
+            // Move media to filesystem
+            for r in db.media.mediaid_file.iter() {
+                let (key, content) = r?;
+                let path = db.globals.get_media_file(&key);
+                let mut file = fs::File::create(path)?;
+                file.write_all(&content)?;
+                db.media.mediaid_file.remove(&key)?;
+                db.media.mediaid_file.insert(&key, vec![])?;
+            }
+
+            db.globals.bump_database_version(3)?;
+
+            info!("Migration: 2 -> 3 finished");
+        }
         // This data is probably outdated
         db.rooms.edus.presenceid_presence.clear()?;
 

src/database/globals.rs

@@ -7,6 +7,8 @@ use ruma::{
 use rustls::{ServerCertVerifier, WebPKIVerifier};
 use std::{
     collections::{BTreeMap, HashMap},
+    fs,
+    path::PathBuf,
     sync::{Arc, RwLock},
     time::{Duration, Instant},
 };
@@ -130,7 +132,7 @@ impl Globals {
             .as_ref()
             .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static());
 
-        Ok(Self {
+        let s = Self {
             globals,
             config,
             keypair: Arc::new(keypair),
@@ -145,7 +147,11 @@ impl Globals {
             bad_event_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())),
             bad_signature_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())),
             servername_ratelimiter: Arc::new(RwLock::new(BTreeMap::new())),
-        })
+        };
+
+        fs::create_dir_all(s.get_media_folder())?;
+
+        Ok(s)
     }
 
     /// Returns this server's keypair.
@@ -264,4 +270,19 @@ impl Globals {
         self.globals.insert("version", &new_version.to_be_bytes())?;
         Ok(())
     }
+
+    pub fn get_media_folder(&self) -> PathBuf {
+        let mut r = PathBuf::new();
+        r.push(self.config.database_path.clone());
+        r.push("media");
+        r
+    }
+
+    pub fn get_media_file(&self, key: &[u8]) -> PathBuf {
+        let mut r = PathBuf::new();
+        r.push(self.config.database_path.clone());
+        r.push("media");
+        r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD));
+        r
+    }
 }

src/database/media.rs

@@ -1,7 +1,9 @@
+use crate::database::globals::Globals;
 use image::{imageops::FilterType, GenericImageView};
 
 use crate::{utils, Error, Result};
 use std::mem;
+use tokio::{fs::File, io::AsyncReadExt, io::AsyncWriteExt};
 
 pub struct FileMeta {
     pub content_disposition: Option<String>,
@@ -15,10 +17,11 @@ pub struct Media {
 }
 
 impl Media {
-    /// Uploads or replaces a file.
-    pub fn create(
+    /// Uploads a file.
+    pub async fn create(
         &self,
         mxc: String,
+        globals: &Globals,
         content_disposition: &Option<&str>,
         content_type: &Option<&str>,
         file: &[u8],
@@ -42,15 +45,19 @@ impl Media {
                 .unwrap_or_default(),
         );
 
-        self.mediaid_file.insert(key, file)?;
+        let path = globals.get_media_file(&key);
+        let mut f = File::create(path).await?;
+        f.write_all(file).await?;
 
+        self.mediaid_file.insert(key, vec![])?;
         Ok(())
     }
 
     /// Uploads or replaces a file thumbnail.
-    pub fn upload_thumbnail(
+    pub async fn upload_thumbnail(
         &self,
         mxc: String,
+        globals: &Globals,
         content_disposition: &Option<String>,
         content_type: &Option<String>,
         width: u32,
@@ -76,21 +83,28 @@ impl Media {
                 .unwrap_or_default(),
         );
 
-        self.mediaid_file.insert(key, file)?;
+        let path = globals.get_media_file(&key);
+        let mut f = File::create(path).await?;
+        f.write_all(file).await?;
+
+        self.mediaid_file.insert(key, vec![])?;
 
         Ok(())
     }
 
     /// Downloads a file.
-    pub fn get(&self, mxc: &str) -> Result<Option<FileMeta>> {
+    pub async fn get(&self, globals: &Globals, mxc: &str) -> Result<Option<FileMeta>> {
         let mut prefix = mxc.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail
         prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
         prefix.push(0xff);
 
-        if let Some(r) = self.mediaid_file.scan_prefix(&prefix).next() {
-            let (key, file) = r?;
+        if let Some(r) = self.mediaid_file.scan_prefix(&prefix).keys().next() {
+            let key = r?;
+            let path = globals.get_media_file(&key);
+            let mut file = vec![];
+            File::open(path).await?.read_to_end(&mut file).await?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
@@ -121,7 +135,7 @@ impl Media {
             Ok(Some(FileMeta {
                 content_disposition,
                 content_type,
-                file: file.to_vec(),
+                file,
             }))
         } else {
             Ok(None)
@@ -151,7 +165,13 @@ impl Media {
     /// - Server creates the thumbnail and sends it to the user
     ///
     /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
-    pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result<Option<FileMeta>> {
+    pub async fn get_thumbnail(
+        &self,
+        mxc: String,
+        globals: &Globals,
+        width: u32,
+        height: u32,
+    ) -> Result<Option<FileMeta>> {
         let (width, height, crop) = self
             .thumbnail_properties(width, height)
             .unwrap_or((0, 0, false)); // 0, 0 because that's the original file
@@ -169,9 +189,17 @@ impl Media {
         original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
         original_prefix.push(0xff);
 
-        if let Some(r) = self.mediaid_file.scan_prefix(&thumbnail_prefix).next() {
+        if let Some(r) = self
+            .mediaid_file
+            .scan_prefix(&thumbnail_prefix)
+            .keys()
+            .next()
+        {
             // Using saved thumbnail
-            let (key, file) = r?;
+            let key = r?;
+            let path = globals.get_media_file(&key);
+            let mut file = vec![];
+            File::open(path).await?.read_to_end(&mut file).await?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
@@ -202,10 +230,19 @@ impl Media {
                 content_type,
                 file: file.to_vec(),
             }))
-        } else if let Some(r) = self.mediaid_file.scan_prefix(&original_prefix).next() {
+        } else if let Some(r) = self
+            .mediaid_file
+            .scan_prefix(&original_prefix)
+            .keys()
+            .next()
+        {
             // Generate a thumbnail
 
-            let (key, file) = r?;
+            let key = r?;
+            let path = globals.get_media_file(&key);
+            let mut file = vec![];
+            File::open(path).await?.read_to_end(&mut file).await?;
+
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
@@ -302,7 +339,11 @@ impl Media {
                     widthheight,
                 );
 
-                self.mediaid_file.insert(thumbnail_key, &*thumbnail_bytes)?;
+                let path = globals.get_media_file(&thumbnail_key);
+                let mut f = File::create(path).await?;
+                f.write_all(&thumbnail_bytes).await?;
+
+                self.mediaid_file.insert(thumbnail_key, vec![])?;
 
                 Ok(Some(FileMeta {
                     content_disposition,

src/error.rs

@@ -40,6 +40,11 @@ pub enum Error {
     },
     #[error("{0}")]
     FederationError(Box<ServerName>, RumaError),
+    #[error("Could not do this io: {source}")]
+    IoError {
+        #[from]
+        source: std::io::Error,
+    },
     #[error("{0}")]
     BadServerResponse(&'static str),
     #[error("{0}")]