Upgrade Ruma

Cargo.lock

@@ -146,25 +146,6 @@ dependencies = [
  "serde",
 ]
 
-[[package]]
-name = "bindgen"
-version = "0.59.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2bd2a9a458e8f4304c52c43ebb0cfbd520289f8379a52e329a38afda99bf8eb8"
-dependencies = [
- "bitflags",
- "cexpr",
- "clang-sys",
- "lazy_static",
- "lazycell",
- "peeking_take_while",
- "proc-macro2",
- "quote",
- "regex",
- "rustc-hash",
- "shlex",
-]
-
 [[package]]
 name = "bitflags"
 version = "1.3.2"
@@ -224,15 +205,6 @@ dependencies = [
  "jobserver",
 ]
 
-[[package]]
-name = "cexpr"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6fac387a98bb7c37292057cffc56d62ecb629900026402633ae9160df93a8766"
-dependencies = [
- "nom",
-]
-
 [[package]]
 name = "cfg-if"
 version = "0.1.10"
@@ -258,17 +230,6 @@ dependencies = [
  "winapi",
 ]
 
-[[package]]
-name = "clang-sys"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "fa66045b9cb23c2e9c1520732030608b02ee07e5cfaa5a521ec15ded7fa24c90"
-dependencies = [
- "glob",
- "libc",
- "libloading",
-]
-
 [[package]]
 name = "color_quant"
 version = "1.1.0"
@@ -298,7 +259,6 @@ dependencies = [
  "reqwest",
  "ring",
  "rocket",
- "rocksdb",
  "ruma",
  "rusqlite",
  "rust-argon2",
@@ -978,7 +938,7 @@ checksum = "527e8c9ac747e28542699a951517aa9a6945af506cd1f2e1b53a576c17b6cc11"
 dependencies = [
  "bytes",
  "fnv",
- "itoa",
+ "itoa 0.4.8",
 ]
 
 [[package]]
@@ -1019,7 +979,7 @@ dependencies = [
  "http-body",
  "httparse",
  "httpdate",
- "itoa",
+ "itoa 0.4.8",
  "pin-project-lite",
  "socket2 0.4.1",
  "tokio",
@@ -1154,6 +1114,12 @@ version = "0.4.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
 
+[[package]]
+name = "itoa"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1aab8fc367588b89dcee83ab0fd66b72b50b72fa1904d7095045ace2b0c81c35"
+
 [[package]]
 name = "jobserver"
 version = "0.1.24"
@@ -1207,40 +1173,12 @@ version = "1.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"
 
-[[package]]
-name = "lazycell"
-version = "1.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55"
-
 [[package]]
 name = "libc"
 version = "0.2.101"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"
 
-[[package]]
-name = "libloading"
-version = "0.7.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "afe203d669ec979b7128619bae5a63b7b42e9203c1b29146079ee05e2f604b52"
-dependencies = [
- "cfg-if 1.0.0",
- "winapi",
-]
-
-[[package]]
-name = "librocksdb-sys"
-version = "6.20.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c309a9d2470844aceb9a4a098cf5286154d20596868b75a6b36357d2bb9ca25d"
-dependencies = [
- "bindgen",
- "cc",
- "glob",
- "libc",
-]
-
 [[package]]
 name = "libsqlite3-sys"
 version = "0.22.2"
@@ -1357,12 +1295,6 @@ version = "0.3.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "2a60c7ce501c71e03a9c9c0d35b861413ae925bd979cc7a4e30d060069aaac8d"
 
-[[package]]
-name = "minimal-lexical"
-version = "0.2.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "68354c5c6bd36d73ff3feceb05efa59b6acb7626617f4962be322a825e61f79a"
-
 [[package]]
 name = "miniz_oxide"
 version = "0.3.7"
@@ -1414,17 +1346,6 @@ dependencies = [
  "version_check",
 ]
 
-[[package]]
-name = "nom"
-version = "7.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "1b1d11e1ef389c76fe5b81bcaf2ea32cf88b62bc494e19f493d0b30e7a930109"
-dependencies = [
- "memchr",
- "minimal-lexical",
- "version_check",
-]
-
 [[package]]
 name = "ntapi"
 version = "0.3.6"
@@ -1624,12 +1545,6 @@ dependencies = [
  "syn",
 ]
 
-[[package]]
-name = "peeking_take_while"
-version = "0.1.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19b17cddbe7ec3f8bc800887bab5e717348c95ea2ca0b1bf0837fb964dc67099"
-
 [[package]]
 name = "pem"
 version = "0.8.3"
@@ -2072,20 +1987,10 @@ dependencies = [
  "uncased",
 ]
 
-[[package]]
-name = "rocksdb"
-version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c749134fda8bfc90d0de643d59bfc841dcb3ac8a1062e12b6754bd60235c48b3"
-dependencies = [
- "libc",
- "librocksdb-sys",
-]
-
 [[package]]
 name = "ruma"
 version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "assign",
  "js_int",
@@ -2106,7 +2011,7 @@ dependencies = [
 [[package]]
 name = "ruma-api"
 version = "0.18.5"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "bytes",
  "http",
@@ -2122,7 +2027,7 @@ dependencies = [
 [[package]]
 name = "ruma-api-macros"
 version = "0.18.5"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -2133,7 +2038,7 @@ dependencies = [
 [[package]]
 name = "ruma-appservice-api"
 version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "ruma-api",
  "ruma-common",
@@ -2147,7 +2052,7 @@ dependencies = [
 [[package]]
 name = "ruma-client-api"
 version = "0.12.3"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "assign",
  "bytes",
@@ -2167,7 +2072,7 @@ dependencies = [
 [[package]]
 name = "ruma-common"
 version = "0.6.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "indexmap",
  "js_int",
@@ -2182,7 +2087,7 @@ dependencies = [
 [[package]]
 name = "ruma-events"
 version = "0.24.6"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "indoc",
  "js_int",
@@ -2198,7 +2103,7 @@ dependencies = [
 [[package]]
 name = "ruma-events-macros"
 version = "0.24.6"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -2209,7 +2114,7 @@ dependencies = [
 [[package]]
 name = "ruma-federation-api"
 version = "0.3.1"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "js_int",
  "ruma-api",
@@ -2224,7 +2129,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers"
 version = "0.20.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "percent-encoding",
  "rand 0.8.4",
@@ -2238,7 +2143,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-macros"
 version = "0.20.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "quote",
  "ruma-identifiers-validation",
@@ -2248,7 +2153,7 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-validation"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "thiserror",
 ]
@@ -2256,7 +2161,7 @@ dependencies = [
 [[package]]
 name = "ruma-identity-service-api"
 version = "0.3.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "js_int",
  "ruma-api",
@@ -2269,7 +2174,7 @@ dependencies = [
 [[package]]
 name = "ruma-push-gateway-api"
 version = "0.3.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "js_int",
  "ruma-api",
@@ -2284,11 +2189,11 @@ dependencies = [
 [[package]]
 name = "ruma-serde"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "bytes",
  "form_urlencoded",
- "itoa",
+ "itoa 0.4.8",
  "js_int",
  "ruma-serde-macros",
  "serde",
@@ -2298,7 +2203,7 @@ dependencies = [
 [[package]]
 name = "ruma-serde-macros"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "proc-macro-crate",
  "proc-macro2",
@@ -2309,7 +2214,7 @@ dependencies = [
 [[package]]
 name = "ruma-signatures"
 version = "0.9.0"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "base64 0.13.0",
  "ed25519-dalek",
@@ -2326,7 +2231,7 @@ dependencies = [
 [[package]]
 name = "ruma-state-res"
 version = "0.4.1"
-source = "git+https://github.com/ruma/ruma?rev=16f031fabb7871fcd738b0f25391193ee4ca28a9#16f031fabb7871fcd738b0f25391193ee4ca28a9"
+source = "git+https://github.com/ruma/ruma?rev=7cf3abbaf02995b03db74429090ca5af1cd71edc#7cf3abbaf02995b03db74429090ca5af1cd71edc"
 dependencies = [
  "itertools 0.10.1",
  "js_int",
@@ -2367,12 +2272,6 @@ dependencies = [
  "crossbeam-utils 0.8.5",
 ]
 
-[[package]]
-name = "rustc-hash"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "08d43f7aa6b08d49f382cde6a7982047c3426db949b1424bc4b7ec9ae12c6ce2"
-
 [[package]]
 name = "rustc_version"
 version = "0.2.3"
@@ -2511,11 +2410,11 @@ dependencies = [
 
 [[package]]
 name = "serde_json"
-version = "1.0.67"
+version = "1.0.73"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a7f9e390c27c3c0ce8bc5d725f6e4d30a29d26659494aa4b17535f7522c5c950"
+checksum = "bcbd0344bc6533bc7ec56df11d42fb70f1b912351c0825ccb7211b59d8af7cf5"
 dependencies = [
- "itoa",
+ "itoa 1.0.1",
  "ryu",
  "serde",
 ]
@@ -2527,7 +2426,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "edfa57a7f8d9c1d260a549e7224100f6c43d43f9103e06dd8b4095a9b2b43ce9"
 dependencies = [
  "form_urlencoded",
- "itoa",
+ "itoa 0.4.8",
  "ryu",
  "serde",
 ]
@@ -2585,12 +2484,6 @@ dependencies = [
  "lazy_static",
 ]
 
-[[package]]
-name = "shlex"
-version = "1.1.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43b2853a4d09f215c24cc5489c992ce46052d359b5109343cbafbf26bc62f8a3"
-
 [[package]]
 name = "signal-hook-registry"
 version = "1.4.0"

Cargo.toml

@@ -19,7 +19,7 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request
 
 # Used for matrix spec type definitions and helpers
 #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
-ruma = { git = "https://github.com/ruma/ruma", rev = "16f031fabb7871fcd738b0f25391193ee4ca28a9", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+ruma = { git = "https://github.com/ruma/ruma", rev = "7cf3abbaf02995b03db74429090ca5af1cd71edc", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 
@@ -36,7 +36,7 @@ http = "0.2.4"
 # Used to find data directory for default db path
 directories = "3.0.2"
 # Used for ruma wrapper
-serde_json = { version = "1.0.67", features = ["raw_value"] }
+serde_json = { version = "1.0.70", features = ["raw_value"] }
 # Used for appservice registration files
 serde_yaml = "0.8.20"
 # Used for pdu definition
@@ -78,18 +78,16 @@ crossbeam = { version = "0.8.1", optional = true }
 num_cpus = "1.13.0"
 threadpool = "1.8.1"
 heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true }
-rocksdb = { version = "0.16.0", features = ["multi-threaded-cf"], optional = true }
 thread_local = "1.1.3"
 # used for TURN server authentication
 hmac = "0.11.0"
 sha-1 = "0.9.8"
 
 [features]
-default = ["conduit_bin", "backend_rocksdb"]
+default = ["conduit_bin", "backend_sqlite"]
 backend_sled = ["sled"]
 backend_sqlite = ["sqlite"]
 backend_heed = ["heed", "crossbeam"]
-backend_rocksdb = ["rocksdb"]
 sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"]
 conduit_bin = [] # TODO: add rocket to this when it is optional
 

src/client_server/keys.rs

@@ -15,7 +15,7 @@ use ruma::{
         },
         federation,
     },
-    encryption::UnsignedDeviceInfo,
+    serde::Raw,
     DeviceId, DeviceKeyAlgorithm, UserId,
 };
 use serde_json::json;
@@ -42,16 +42,9 @@ pub async fn upload_keys_route(
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
     let sender_device = body.sender_device.as_ref().expect("user is authenticated");
 
-    if let Some(one_time_keys) = &body.one_time_keys {
+    for (key_key, key_value) in &body.one_time_keys {
-        for (key_key, key_value) in one_time_keys {
+        db.users
-            db.users.add_one_time_key(
+            .add_one_time_key(sender_user, sender_device, key_key, key_value, &db.globals)?;
-                sender_user,
-                sender_device,
-                key_key,
-                key_value,
-                &db.globals,
-            )?;
-        }
     }
 
     if let Some(device_keys) = &body.device_keys {
@@ -350,10 +343,8 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
                             Error::bad_database("all_device_keys contained nonexistent device.")
                         })?;
 
-                    keys.unsigned = UnsignedDeviceInfo {
+                    add_unsigned_device_display_name(&mut keys, metadata)
-                        device_display_name: metadata.display_name,
+                        .map_err(|_| Error::bad_database("invalid device keys in database"))?;
-                    };
-
                     container.insert(device_id, keys);
                 }
             }
@@ -369,10 +360,8 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
                         ),
                     )?;
 
-                    keys.unsigned = UnsignedDeviceInfo {
+                    add_unsigned_device_display_name(&mut keys, metadata)
-                        device_display_name: metadata.display_name,
+                        .map_err(|_| Error::bad_database("invalid device keys in database"))?;
-                    };
-
                     container.insert(device_id.to_owned(), keys);
                 }
                 device_keys.insert(user_id.to_owned(), container);
@@ -441,6 +430,24 @@ pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
     })
 }
 
+fn add_unsigned_device_display_name(
+    keys: &mut Raw<ruma::encryption::DeviceKeys>,
+    metadata: ruma::api::client::r0::device::Device,
+) -> serde_json::Result<()> {
+    if let Some(display_name) = metadata.display_name {
+        let mut object = keys.deserialize_as::<serde_json::Map<String, serde_json::Value>>()?;
+
+        let unsigned = object.entry("unsigned").or_insert_with(|| json!({}));
+        if let serde_json::Value::Object(unsigned_object) = unsigned {
+            unsigned_object.insert("device_display_name".to_owned(), display_name.into());
+        }
+
+        *keys = Raw::from_json(serde_json::value::to_raw_value(&object)?);
+    }
+
+    Ok(())
+}
+
 pub(crate) async fn claim_keys_helper(
     one_time_keys_input: &BTreeMap<Box<UserId>, BTreeMap<Box<DeviceId>, DeviceKeyAlgorithm>>,
     db: &Database,

src/client_server/sync.rs

@@ -762,6 +762,8 @@ async fn sync_helper(
                 .users
                 .get_to_device_events(&sender_user, &sender_device)?,
         },
+        // Fallback keys are not yet supported
+        device_unused_fallback_key_types: None,
     };
 
     // TODO: Retry the endpoint instead of returning (waiting for #118)

src/database.rs

@@ -154,9 +154,6 @@ pub type Engine = abstraction::sqlite::Engine;
 #[cfg(feature = "heed")]
 pub type Engine = abstraction::heed::Engine;
 
-#[cfg(feature = "rocksdb")]
-pub type Engine = abstraction::rocksdb::Engine;
-
 pub struct Database {
     _db: Arc<Engine>,
     pub globals: globals::Globals,

src/database/abstraction.rs

@@ -12,9 +12,6 @@ pub mod sqlite;
 #[cfg(feature = "heed")]
 pub mod heed;
 
-#[cfg(feature = "rocksdb")]
-pub mod rocksdb;
-
 pub trait DatabaseEngine: Sized {
     fn open(config: &Config) -> Result<Arc<Self>>;
     fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>>;

src/database/abstraction/rocksdb.rs

@@ -1,234 +0,0 @@
-use super::super::Config;
-use crate::{utils, Result};
-use std::{future::Future, pin::Pin, sync::Arc};
-use std::{
-    collections::{hash_map, HashMap},
-    sync::RwLock};
-use super::{DatabaseEngine, Tree};
-use tokio::sync::watch;
-
-pub struct Engine {
-    rocks: rocksdb::DBWithThreadMode<rocksdb::MultiThreaded>,
-    old_cfs: Vec<String>,
-}
-
-pub struct RocksDbEngineTree<'a> {
-    db: Arc<Engine>,
-    name: &'a str,
-    watchers: RwLock<HashMap<Vec<u8>, (watch::Sender<()>, watch::Receiver<()>)>>,
-    write_lock: RwLock<()>,
-}
-
-impl DatabaseEngine for Engine {
-    fn open(config: &Config) -> Result<Arc<Self>> {
-        let mut db_opts = rocksdb::Options::default();
-        db_opts.create_if_missing(true);
-        db_opts.set_max_open_files(512);
-        db_opts.set_compaction_style(rocksdb::DBCompactionStyle::Level);
-        db_opts.set_compression_type(rocksdb::DBCompressionType::Zstd);
-        db_opts.set_target_file_size_base(2 << 22);
-        db_opts.set_max_bytes_for_level_base(2 << 24);
-        db_opts.set_max_bytes_for_level_multiplier(2.0);
-        db_opts.set_num_levels(8);
-        db_opts.set_write_buffer_size(2 << 27);
-
-        let rocksdb_cache =
-            rocksdb::Cache::new_lru_cache((config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize)
-                .unwrap();
-
-        let mut block_based_options = rocksdb::BlockBasedOptions::default();
-        block_based_options.set_block_size(2 << 19);
-        block_based_options.set_block_cache(&rocksdb_cache);
-        db_opts.set_block_based_table_factory(&block_based_options);
-
-        let cfs = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::list_cf(
-            &db_opts,
-            &config.database_path,
-        )
-        .unwrap_or_default();
-
-        let db = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::open_cf_descriptors(
-            &db_opts,
-            &config.database_path,
-            cfs.iter().map(|name| {
-                let mut options = rocksdb::Options::default();
-                let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1);
-                options.set_prefix_extractor(prefix_extractor);
-
-                rocksdb::ColumnFamilyDescriptor::new(name, options)
-            }),
-        )?;
-
-        Ok(Arc::new(Engine {
-            rocks: db,
-            old_cfs: cfs,
-        }))
-    }
-
-    fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> {
-        if !self.old_cfs.contains(&name.to_owned()) {
-            // Create if it didn't exist
-            let mut options = rocksdb::Options::default();
-            let prefix_extractor = rocksdb::SliceTransform::create_fixed_prefix(1);
-            options.set_prefix_extractor(prefix_extractor);
-
-            let _ = self.rocks.create_cf(name, &options);
-            println!("created cf");
-        }
-
-        Ok(Arc::new(RocksDbEngineTree {
-            name,
-            db: Arc::clone(self),
-            watchers: RwLock::new(HashMap::new()),
-            write_lock: RwLock::new(()),
-        }))
-    }
-
-    fn flush(self: &Arc<Self>) -> Result<()> {
-        // TODO?
-        Ok(())
-    }
-}
-
-impl RocksDbEngineTree<'_> {
-    fn cf(&self) -> rocksdb::BoundColumnFamily<'_> {
-        self.db.rocks.cf_handle(self.name).unwrap()
-    }
-}
-
-impl Tree for RocksDbEngineTree<'_> {
-    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
-        Ok(self.db.rocks.get_cf(self.cf(), key)?)
-    }
-
-    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
-        let lock = self.write_lock.read().unwrap();
-
-        let result = self.db.rocks.put_cf(self.cf(), key, value)?;
-
-        drop(lock);
-
-        let watchers = self.watchers.read().unwrap();
-        let mut triggered = Vec::new();
-
-        for length in 0..=key.len() {
-            if watchers.contains_key(&key[..length]) {
-                triggered.push(&key[..length]);
-            }
-        }
-
-        drop(watchers);
-
-        if !triggered.is_empty() {
-            let mut watchers = self.watchers.write().unwrap();
-            for prefix in triggered {
-                if let Some(tx) = watchers.remove(prefix) {
-                    let _ = tx.0.send(());
-                }
-            }
-        };
-
-        Ok(result)
-    }
-
-    fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
-        for (key, value) in iter {
-            self.db.rocks.put_cf(self.cf(), key, value)?;
-        }
-
-        Ok(())
-    }
-
-    fn remove(&self, key: &[u8]) -> Result<()> {
-        Ok(self.db.rocks.delete_cf(self.cf(), key)?)
-    }
-
-    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
-        Box::new(
-            self.db
-                .rocks
-                .iterator_cf(self.cf(), rocksdb::IteratorMode::Start)
-                .map(|(k, v)| (Vec::from(k), Vec::from(v))),
-        )
-    }
-
-    fn iter_from<'a>(
-        &'a self,
-        from: &[u8],
-        backwards: bool,
-    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
-        Box::new(
-            self.db
-                .rocks
-                .iterator_cf(
-                    self.cf(),
-                    rocksdb::IteratorMode::From(
-                        from,
-                        if backwards {
-                            rocksdb::Direction::Reverse
-                        } else {
-                            rocksdb::Direction::Forward
-                        },
-                    ),
-                )
-                .map(|(k, v)| (Vec::from(k), Vec::from(v))),
-        )
-    }
-
-    fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
-        let lock = self.write_lock.write().unwrap();
-
-        let old = self.db.rocks.get_cf(self.cf(), &key)?;
-        let new = utils::increment(old.as_deref()).unwrap();
-        self.db.rocks.put_cf(self.cf(), key, &new)?;
-
-        drop(lock);
-        Ok(new)
-    }
-
-    fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
-        let lock = self.write_lock.write().unwrap();
-
-        for key in iter {
-            let old = self.db.rocks.get_cf(self.cf(), &key)?;
-            let new = utils::increment(old.as_deref()).unwrap();
-            self.db.rocks.put_cf(self.cf(), key, new)?;
-        }
-
-        drop(lock);
-
-        Ok(())
-    }
-
-    fn scan_prefix<'a>(
-        &'a self,
-        prefix: Vec<u8>,
-    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
-        Box::new(
-            self.db
-                .rocks
-                .iterator_cf(
-                    self.cf(),
-                    rocksdb::IteratorMode::From(&prefix, rocksdb::Direction::Forward),
-                )
-                .map(|(k, v)| (Vec::from(k), Vec::from(v)))
-                .take_while(move |(k, _)| k.starts_with(&prefix)),
-        )
-    }
-
-    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
-        let mut rx = match self.watchers.write().unwrap().entry(prefix.to_vec()) {
-            hash_map::Entry::Occupied(o) => o.get().1.clone(),
-            hash_map::Entry::Vacant(v) => {
-                let (tx, rx) = tokio::sync::watch::channel(());
-                v.insert((tx, rx.clone()));
-                rx
-            }
-        };
-
-        Box::pin(async move {
-            // Tx is never destroyed
-            rx.changed().await.unwrap();
-        })
-    }
-}

src/database/abstraction/sqlite.rs

@@ -134,7 +134,7 @@ type TupleOfBytes = (Vec<u8>, Vec<u8>);
 impl SqliteTable {
     #[tracing::instrument(skip(self, guard, key))]
     fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result<Option<Vec<u8>>> {
-        dbg!(&self.name);
+        //dbg!(&self.name);
         Ok(guard
             .prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())?
             .query_row([key], |row| row.get(0))
@@ -143,7 +143,7 @@ impl SqliteTable {
 
     #[tracing::instrument(skip(self, guard, key, value))]
     fn insert_with_guard(&self, guard: &Connection, key: &[u8], value: &[u8]) -> Result<()> {
-        dbg!(&self.name);
+        //dbg!(&self.name);
         guard.execute(
             format!(
                 "INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)",
@@ -170,14 +170,14 @@ impl SqliteTable {
 
         let statement_ref = NonAliasingBox(statement);
 
-        let name = self.name.clone();
+        //let name = self.name.clone();
 
         let iterator = Box::new(
             statement
                 .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
                 .unwrap()
                 .map(move |r| {
-                    dbg!(&name);
+                    //dbg!(&name);
                     r.unwrap()
                 }),
         );
@@ -285,7 +285,7 @@ impl Tree for SqliteTable {
         let guard = self.engine.read_lock_iterator();
         let from = from.to_vec(); // TODO change interface?
 
-        let name = self.name.clone();
+        //let name = self.name.clone();
 
         if backwards {
             let statement = Box::leak(Box::new(
@@ -304,7 +304,7 @@ impl Tree for SqliteTable {
                     .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
                     .unwrap()
                     .map(move |r| {
-                        dbg!(&name);
+                        //dbg!(&name);
                         r.unwrap()
                     }),
             );
@@ -329,7 +329,7 @@ impl Tree for SqliteTable {
                     .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
                     .unwrap()
                     .map(move |r| {
-                        dbg!(&name);
+                        //dbg!(&name);
                         r.unwrap()
                     }),
             );

src/database/key_backups.rs

@@ -4,8 +4,10 @@ use ruma::{
         error::ErrorKind,
         r0::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup},
     },
+    serde::Raw,
     RoomId, UserId,
 };
+use serde_json::json;
 use std::{collections::BTreeMap, sync::Arc};
 
 use super::abstraction::Tree;
@@ -20,7 +22,7 @@ impl KeyBackups {
     pub fn create_backup(
         &self,
         user_id: &UserId,
-        backup_metadata: &BackupAlgorithm,
+        backup_metadata: &Raw<BackupAlgorithm>,
         globals: &super::globals::Globals,
     ) -> Result<String> {
         let version = globals.next_count()?.to_string();
@@ -59,7 +61,7 @@ impl KeyBackups {
         &self,
         user_id: &UserId,
         version: &str,
-        backup_metadata: &BackupAlgorithm,
+        backup_metadata: &Raw<BackupAlgorithm>,
         globals: &super::globals::Globals,
     ) -> Result<String> {
         let mut key = user_id.as_bytes().to_vec();
@@ -73,12 +75,8 @@ impl KeyBackups {
             ));
         }
 
-        self.backupid_algorithm.insert(
+        self.backupid_algorithm
-            &key,
+            .insert(&key, backup_metadata.json().get().as_bytes())?;
-            serde_json::to_string(backup_metadata)
-                .expect("BackupAlgorithm::to_string always works")
-                .as_bytes(),
-        )?;
         self.backupid_etag
             .insert(&key, &globals.next_count()?.to_be_bytes())?;
         Ok(version.to_owned())
@@ -105,7 +103,10 @@ impl KeyBackups {
             .transpose()
     }
 
-    pub fn get_latest_backup(&self, user_id: &UserId) -> Result<Option<(String, BackupAlgorithm)>> {
+    pub fn get_latest_backup(
+        &self,
+        user_id: &UserId,
+    ) -> Result<Option<(String, Raw<BackupAlgorithm>)>> {
         let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         let mut last_possible_key = prefix.clone();
@@ -133,7 +134,11 @@ impl KeyBackups {
             .transpose()
     }
 
-    pub fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Option<BackupAlgorithm>> {
+    pub fn get_backup(
+        &self,
+        user_id: &UserId,
+        version: &str,
+    ) -> Result<Option<Raw<BackupAlgorithm>>> {
         let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(version.as_bytes());
@@ -152,7 +157,7 @@ impl KeyBackups {
         version: &str,
         room_id: &RoomId,
         session_id: &str,
-        key_data: &KeyBackupData,
+        key_data: &Raw<KeyBackupData>,
         globals: &super::globals::Globals,
     ) -> Result<()> {
         let mut key = user_id.as_bytes().to_vec();
@@ -174,10 +179,8 @@ impl KeyBackups {
         key.push(0xff);
         key.extend_from_slice(session_id.as_bytes());
 
-        self.backupkeyid_backup.insert(
+        self.backupkeyid_backup
-            &key,
+            .insert(&key, key_data.json().get().as_bytes())?;
-            &serde_json::to_vec(&key_data).expect("KeyBackupData::to_vec always works"),
-        )?;
 
         Ok(())
     }
@@ -209,13 +212,13 @@ impl KeyBackups {
         &self,
         user_id: &UserId,
         version: &str,
-    ) -> Result<BTreeMap<Box<RoomId>, RoomKeyBackup>> {
+    ) -> Result<BTreeMap<Box<RoomId>, Raw<RoomKeyBackup>>> {
         let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
         prefix.push(0xff);
 
-        let mut rooms = BTreeMap::<Box<RoomId>, RoomKeyBackup>::new();
+        let mut rooms = BTreeMap::<Box<RoomId>, Raw<RoomKeyBackup>>::new();
 
         for result in self
             .backupkeyid_backup
@@ -241,7 +244,7 @@ impl KeyBackups {
                     Error::bad_database("backupkeyid_backup room_id is invalid room id.")
                 })?;
 
-                let key_data = serde_json::from_slice(&value).map_err(|_| {
+                let key_data: serde_json::Value = serde_json::from_slice(&value).map_err(|_| {
                     Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
                 })?;
 
@@ -249,13 +252,25 @@ impl KeyBackups {
             })
         {
             let (room_id, session_id, key_data) = result?;
-            rooms
+            let room_key_backup = rooms.entry(room_id).or_insert_with(|| {
-                .entry(room_id)
+                Raw::new(&RoomKeyBackup {
-                .or_insert_with(|| RoomKeyBackup {
                     sessions: BTreeMap::new(),
                 })
-                .sessions
+                .expect("RoomKeyBackup serialization")
-                .insert(session_id, key_data);
+            });
+
+            let mut object = room_key_backup
+                .deserialize_as::<serde_json::Map<String, serde_json::Value>>()
+                .map_err(|_| Error::bad_database("RoomKeyBackup is not an object"))?;
+
+            let sessions = object.entry("session").or_insert_with(|| json!({}));
+            if let serde_json::Value::Object(unsigned_object) = sessions {
+                unsigned_object.insert(session_id, key_data);
+            }
+
+            *room_key_backup = Raw::from_json(
+                serde_json::value::to_raw_value(&object).expect("Value => RawValue serialization"),
+            );
         }
 
         Ok(rooms)
@@ -266,7 +281,7 @@ impl KeyBackups {
         user_id: &UserId,
         version: &str,
         room_id: &RoomId,
-    ) -> Result<BTreeMap<String, KeyBackupData>> {
+    ) -> Result<BTreeMap<String, Raw<KeyBackupData>>> {
         let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
@@ -304,7 +319,7 @@ impl KeyBackups {
         version: &str,
         room_id: &RoomId,
         session_id: &str,
-    ) -> Result<Option<KeyBackupData>> {
+    ) -> Result<Option<Raw<KeyBackupData>>> {
         let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(version.as_bytes());

src/database/users.rs

@@ -8,7 +8,12 @@ use ruma::{
     DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, RoomAliasId, UInt,
     UserId,
 };
-use std::{collections::BTreeMap, convert::TryInto, mem, sync::Arc};
+use std::{
+    collections::BTreeMap,
+    convert::{TryFrom, TryInto},
+    mem,
+    sync::Arc,
+};
 use tracing::warn;
 
 use super::abstraction::Tree;
@@ -359,7 +364,7 @@ impl Users {
         user_id: &UserId,
         device_id: &DeviceId,
         one_time_key_key: &DeviceKeyId,
-        one_time_key_value: &OneTimeKey,
+        one_time_key_value: &Raw<OneTimeKey>,
         globals: &super::globals::Globals,
     ) -> Result<()> {
         let mut key = user_id.as_bytes().to_vec();
@@ -409,7 +414,7 @@ impl Users {
         device_id: &DeviceId,
         key_algorithm: &DeviceKeyAlgorithm,
         globals: &super::globals::Globals,
-    ) -> Result<Option<(Box<DeviceKeyId>, OneTimeKey)>> {
+    ) -> Result<Option<(Box<DeviceKeyId>, Raw<OneTimeKey>)>> {
         let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(device_id.as_bytes());
@@ -480,7 +485,7 @@ impl Users {
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
-        device_keys: &DeviceKeys,
+        device_keys: &Raw<DeviceKeys>,
         rooms: &super::rooms::Rooms,
         globals: &super::globals::Globals,
     ) -> Result<()> {
@@ -509,9 +514,9 @@ impl Users {
     pub fn add_cross_signing_keys(
         &self,
         user_id: &UserId,
-        master_key: &CrossSigningKey,
+        master_key: &Raw<CrossSigningKey>,
-        self_signing_key: &Option<CrossSigningKey>,
+        self_signing_key: &Option<Raw<CrossSigningKey>>,
-        user_signing_key: &Option<CrossSigningKey>,
+        user_signing_key: &Option<Raw<CrossSigningKey>>,
         rooms: &super::rooms::Rooms,
         globals: &super::globals::Globals,
     ) -> Result<()> {
@@ -521,7 +526,12 @@ impl Users {
         prefix.push(0xff);
 
         // Master key
-        let mut master_key_ids = master_key.keys.values();
+        let master_key_map = master_key
+            .deserialize()
+            .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid master key"))?
+            .keys;
+        let mut master_key_ids = master_key_map.values();
+
         let master_key_id = master_key_ids.next().ok_or(Error::BadRequest(
             ErrorKind::InvalidParam,
             "Master key contained no key.",
@@ -537,17 +547,21 @@ impl Users {
         let mut master_key_key = prefix.clone();
         master_key_key.extend_from_slice(master_key_id.as_bytes());
 
-        self.keyid_key.insert(
+        self.keyid_key
-            &master_key_key,
+            .insert(&master_key_key, master_key.json().get().as_bytes())?;
-            &serde_json::to_vec(&master_key).expect("CrossSigningKey::to_vec always works"),
-        )?;
 
         self.userid_masterkeyid
             .insert(user_id.as_bytes(), &master_key_key)?;
 
         // Self-signing key
         if let Some(self_signing_key) = self_signing_key {
-            let mut self_signing_key_ids = self_signing_key.keys.values();
+            let self_signing_key_map = self_signing_key
+                .deserialize()
+                .map_err(|_| {
+                    Error::BadRequest(ErrorKind::InvalidParam, "Invalid self signing key")
+                })?
+                .keys;
+            let mut self_signing_key_ids = self_signing_key_map.values();
             let self_signing_key_id = self_signing_key_ids.next().ok_or(Error::BadRequest(
                 ErrorKind::InvalidParam,
                 "Self signing key contained no key.",
@@ -565,8 +579,7 @@ impl Users {
 
             self.keyid_key.insert(
                 &self_signing_key_key,
-                &serde_json::to_vec(&self_signing_key)
+                self_signing_key.json().get().as_bytes(),
-                    .expect("CrossSigningKey::to_vec always works"),
             )?;
 
             self.userid_selfsigningkeyid
@@ -575,7 +588,13 @@ impl Users {
 
         // User-signing key
         if let Some(user_signing_key) = user_signing_key {
-            let mut user_signing_key_ids = user_signing_key.keys.values();
+            let user_signing_key_map = user_signing_key
+                .deserialize()
+                .map_err(|_| {
+                    Error::BadRequest(ErrorKind::InvalidParam, "Invalid user signing key")
+                })?
+                .keys;
+            let mut user_signing_key_ids = user_signing_key_map.values();
             let user_signing_key_id = user_signing_key_ids.next().ok_or(Error::BadRequest(
                 ErrorKind::InvalidParam,
                 "User signing key contained no key.",
@@ -593,8 +612,7 @@ impl Users {
 
             self.keyid_key.insert(
                 &user_signing_key_key,
-                &serde_json::to_vec(&user_signing_key)
+                user_signing_key.json().get().as_bytes(),
-                    .expect("CrossSigningKey::to_vec always works"),
             )?;
 
             self.userid_usersigningkeyid
@@ -727,7 +745,7 @@ impl Users {
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
-    ) -> Result<Option<DeviceKeys>> {
+    ) -> Result<Option<Raw<DeviceKeys>>> {
         let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(device_id.as_bytes());
@@ -744,25 +762,19 @@ impl Users {
         &self,
         user_id: &UserId,
         allowed_signatures: F,
-    ) -> Result<Option<CrossSigningKey>> {
+    ) -> Result<Option<Raw<CrossSigningKey>>> {
         self.userid_masterkeyid
             .get(user_id.as_bytes())?
             .map_or(Ok(None), |key| {
                 self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
-                    let mut cross_signing_key = serde_json::from_slice::<CrossSigningKey>(&bytes)
+                    let mut cross_signing_key = serde_json::from_slice::<serde_json::Value>(&bytes)
-                        .map_err(|_| {
+                        .map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?;
-                        Error::bad_database("CrossSigningKey in db is invalid.")
+                    clean_signatures(&mut cross_signing_key, user_id, allowed_signatures)?;
-                    })?;
 
-                    // A user is not allowed to see signatures from users other than himself and
+                    Ok(Some(Raw::from_json(
-                    // the target user
+                        serde_json::value::to_raw_value(&cross_signing_key)
-                    cross_signing_key.signatures = cross_signing_key
+                            .expect("Value to RawValue serialization"),
-                        .signatures
+                    )))
-                        .into_iter()
-                        .filter(|(user, _)| allowed_signatures(user))
-                        .collect();
-
-                    Ok(Some(cross_signing_key))
                 })
             })
     }
@@ -772,31 +784,25 @@ impl Users {
         &self,
         user_id: &UserId,
         allowed_signatures: F,
-    ) -> Result<Option<CrossSigningKey>> {
+    ) -> Result<Option<Raw<CrossSigningKey>>> {
         self.userid_selfsigningkeyid
             .get(user_id.as_bytes())?
             .map_or(Ok(None), |key| {
                 self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
-                    let mut cross_signing_key = serde_json::from_slice::<CrossSigningKey>(&bytes)
+                    let mut cross_signing_key = serde_json::from_slice::<serde_json::Value>(&bytes)
-                        .map_err(|_| {
+                        .map_err(|_| Error::bad_database("CrossSigningKey in db is invalid."))?;
-                        Error::bad_database("CrossSigningKey in db is invalid.")
+                    clean_signatures(&mut cross_signing_key, user_id, allowed_signatures)?;
-                    })?;
 
-                    // A user is not allowed to see signatures from users other than himself and
+                    Ok(Some(Raw::from_json(
-                    // the target user
+                        serde_json::value::to_raw_value(&cross_signing_key)
-                    cross_signing_key.signatures = cross_signing_key
+                            .expect("Value to RawValue serialization"),
-                        .signatures
+                    )))
-                        .into_iter()
-                        .filter(|(user, _)| user == user_id || allowed_signatures(user))
-                        .collect();
-
-                    Ok(Some(cross_signing_key))
                 })
             })
     }
 
     #[tracing::instrument(skip(self, user_id))]
-    pub fn get_user_signing_key(&self, user_id: &UserId) -> Result<Option<CrossSigningKey>> {
+    pub fn get_user_signing_key(&self, user_id: &UserId) -> Result<Option<Raw<CrossSigningKey>>> {
         self.userid_usersigningkeyid
             .get(user_id.as_bytes())?
             .map_or(Ok(None), |key| {
@@ -991,3 +997,30 @@ impl Users {
         Ok(())
     }
 }
+
+/// Ensure that a user only sees signatures from themselves and the target user
+fn clean_signatures<F: Fn(&UserId) -> bool>(
+    cross_signing_key: &mut serde_json::Value,
+    user_id: &UserId,
+    allowed_signatures: F,
+) -> Result<(), Error> {
+    if let Some(signatures) = cross_signing_key
+        .get_mut("signatures")
+        .and_then(|v| v.as_object_mut())
+    {
+        // Don't allocate for the full size of the current signatures, but require
+        // at most one resize if nothing is dropped
+        let new_capacity = signatures.len() / 2;
+        for (user, signature) in
+            mem::replace(signatures, serde_json::Map::with_capacity(new_capacity))
+        {
+            let id = <&UserId>::try_from(user.as_str())
+                .map_err(|_| Error::bad_database("Invalid user ID in database."))?;
+            if id == user_id || allowed_signatures(id) {
+                signatures.insert(user, signature);
+            }
+        }
+    }
+
+    Ok(())
+}

src/error.rs

@@ -39,12 +39,6 @@ pub enum Error {
     #[cfg(feature = "heed")]
     #[error("There was a problem with the connection to the heed database: {error}")]
     HeedError { error: String },
-    #[cfg(feature = "rocksdb")]
-    #[error("There was a problem with the connection to the rocksdb database: {source}")]
-    RocksDbError {
-        #[from]
-        source: rocksdb::Error,
-    },
     #[error("Could not generate an image.")]
     ImageError {
         #[from]

src/server_server.rs

@@ -1392,11 +1392,12 @@ async fn upgrade_outlier_to_timeline_pdu(
                 let mut starting_events = Vec::with_capacity(leaf_state.len());
 
                 for (k, id) in leaf_state {
-                    if let Ok(k) = db.rooms.get_statekey_from_short(k) {
+                    let k = db
+                        .rooms
+                        .get_statekey_from_short(k)
+                        .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?;
+
                     state.insert(k, id.clone());
-                    } else {
-                        warn!("Failed to get_statekey_from_short.");
-                    }
                     starting_events.push(id);
                 }
 
@@ -1686,6 +1687,25 @@ async fn upgrade_outlier_to_timeline_pdu(
         // We do this by adding the current state to the list of fork states
         extremity_sstatehashes.remove(&current_sstatehash);
         fork_states.push(current_state_ids);
+        dbg!(&extremity_sstatehashes);
+
+        for (sstatehash, leaf_pdu) in extremity_sstatehashes {
+            let mut leaf_state = db
+                .rooms
+                .state_full_ids(sstatehash)
+                .map_err(|_| "Failed to ask db for room state.".to_owned())?;
+
+            if let Some(state_key) = &leaf_pdu.state_key {
+                let shortstatekey = db
+                    .rooms
+                    .get_or_create_shortstatekey(&leaf_pdu.kind, state_key, &db.globals)
+                    .map_err(|_| "Failed to create shortstatekey.".to_owned())?;
+                leaf_state.insert(shortstatekey, Arc::from(&*leaf_pdu.event_id));
+                // Now it's the state after the pdu
+            }
+
+            fork_states.push(leaf_state);
+        }
 
         // We also add state after incoming event to the fork states
         let mut state_after = state_at_incoming_event.clone();
@@ -1735,16 +1755,11 @@ async fn upgrade_outlier_to_timeline_pdu(
                 .into_iter()
                 .map(|map| {
                     map.into_iter()
-                        .filter_map(|(k, id)| {
+                        .map(|(k, id)| db.rooms.get_statekey_from_short(k).map(|k| (k, id)))
-                            db.rooms
+                        .collect::<Result<StateMap<_>>>()
-                                .get_statekey_from_short(k)
-                                .map(|k| (k, id))
-                                .map_err(|e| warn!("Failed to get_statekey_from_short: {}", e))
-                                .ok()
                 })
-                        .collect::<StateMap<_>>()
+                .collect::<Result<_>>()
-                })
+                .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?;
-                .collect();
 
             let state = match state_res::resolve(
                 room_version_id,
@@ -1856,86 +1871,45 @@ pub(crate) fn fetch_and_handle_outliers<'a>(
             // a. Look in the main timeline (pduid_pdu tree)
             // b. Look at outlier pdu tree
             // (get_pdu_json checks both)
-            if let Ok(Some(local_pdu)) = db.rooms.get_pdu(id) {
+            let local_pdu = db.rooms.get_pdu(id);
+            let pdu = match local_pdu {
+                Ok(Some(pdu)) => {
                     trace!("Found {} in db", id);
-                pdus.push((local_pdu, None));
+                    (pdu, None)
-                continue;
                 }
-
+                Ok(None) => {
                     // c. Ask origin server over federation
-            // We also handle its auth chain here so we don't get a stack overflow in
+                    warn!("Fetching {} over federation.", id);
-            // handle_outlier_pdu.
-            let mut todo_auth_events = vec![Arc::clone(id)];
-            let mut events_in_reverse_order = Vec::new();
-            let mut events_all = HashSet::new();
-            while let Some(next_id) = todo_auth_events.pop() {
-                if events_all.contains(&next_id) {
-                    continue;
-                }
-
-                if let Ok(Some(_)) = db.rooms.get_pdu(&next_id) {
-                    trace!("Found {} in db", id);
-                    continue;
-                }
-
-                warn!("Fetching {} over federation.", next_id);
                     match db
                         .sending
                         .send_federation_request(
                             &db.globals,
                             origin,
-                        get_event::v1::Request { event_id: &next_id },
+                            get_event::v1::Request { event_id: id },
                         )
                         .await
                     {
                         Ok(res) => {
-                        warn!("Got {} over federation", next_id);
+                            warn!("Got {} over federation", id);
                             let (calculated_event_id, value) =
                                 match crate::pdu::gen_event_id_canonical_json(&res.pdu) {
                                     Ok(t) => t,
                                     Err(_) => {
-                                    back_off((*next_id).to_owned());
+                                        back_off((**id).to_owned());
                                         continue;
                                     }
                                 };
 
-                        if calculated_event_id != *next_id {
+                            if calculated_event_id != **id {
                                 warn!("Server didn't return event id we requested: requested: {}, we got {}. Event: {:?}",
-                                next_id, calculated_event_id, &res.pdu);
+                                    id, calculated_event_id, &res.pdu);
                             }
 
-                        if let Some(auth_events) =
+                            // This will also fetch the auth chain
-                            value.get("auth_events").and_then(|c| c.as_array())
-                        {
-                            for auth_event in auth_events {
-                                if let Ok(auth_event) =
-                                    serde_json::from_value(auth_event.clone().into())
-                                {
-                                    let a: Arc<EventId> = auth_event;
-                                    todo_auth_events.push(a);
-                                } else {
-                                    warn!("Auth event id is not valid");
-                                }
-                            }
-                        } else {
-                            warn!("Auth event list invalid");
-                        }
-
-                        events_in_reverse_order.push((next_id.clone(), value));
-                        events_all.insert(next_id);
-                    }
-                    Err(_) => {
-                        warn!("Failed to fetch event: {}", next_id);
-                        back_off((*next_id).to_owned());
-                    }
-                }
-            }
-
-            for (next_id, value) in events_in_reverse_order.iter().rev() {
                             match handle_outlier_pdu(
                                 origin,
                                 create_event,
-                    &next_id,
+                                id,
                                 room_id,
                                 value.clone(),
                                 db,
@@ -1943,17 +1917,27 @@ pub(crate) fn fetch_and_handle_outliers<'a>(
                             )
                             .await
                             {
-                    Ok((pdu, json)) => {
+                                Ok((pdu, json)) => (pdu, Some(json)),
-                        if next_id == id {
+                                Err(e) => {
-                            pdus.push((pdu, Some(json)));
+                                    warn!("Authentication of event {} failed: {:?}", id, e);
+                                    back_off((**id).to_owned());
+                                    continue;
+                                }
+                            }
+                        }
+                        Err(_) => {
+                            warn!("Failed to fetch event: {}", id);
+                            back_off((**id).to_owned());
+                            continue;
+                        }
                     }
                 }
                 Err(e) => {
-                        warn!("Authentication of event {} failed: {:?}", next_id, e);
+                    warn!("Error loading {}: {}", id, e);
-                        back_off((**next_id).to_owned());
+                    continue;
-                    }
-                }
                 }
+            };
+            pdus.push(pdu);
         }
         pdus
     })

src/utils.rs

@@ -29,17 +29,6 @@ pub fn increment(old: Option<&[u8]>) -> Option<Vec<u8>> {
     Some(number.to_be_bytes().to_vec())
 }
 
-#[cfg(feature = "rocksdb")]
-pub fn increment_rocksdb(
-    _new_key: &[u8],
-    old: Option<&[u8]>,
-    _operands: &mut rocksdb::MergeOperands,
-) -> Option<Vec<u8>> {
-    dbg!(_new_key);
-    dbg!(old);
-    increment(old)
-}
-
 pub fn generate_keypair() -> Vec<u8> {
     let mut value = random_string(8).as_bytes().to_vec();
     value.push(0xff);