diff --git a/Cargo.lock b/Cargo.lock index 70d7f4b..3f9d4fb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -254,6 +254,7 @@ dependencies = [ "opentelemetry-jaeger", "parking_lot", "rand 0.8.4", + "rayon", "regex", "reqwest", "ring", @@ -1747,6 +1748,31 @@ dependencies = [ "rand_core 0.6.3", ] +[[package]] +name = "rayon" +version = "1.5.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c06aca804d41dbc8ba42dfd964f0d01334eceb64314b9ecf7c5fad5188a06d90" +dependencies = [ + "autocfg", + "crossbeam-deque", + "either", + "rayon-core", +] + +[[package]] +name = "rayon-core" +version = "1.9.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d78120e2c850279833f1dd3582f730c4ab53ed95aeaaaa862a2a5c71b1656d8e" +dependencies = [ + "crossbeam-channel", + "crossbeam-deque", + "crossbeam-utils 0.8.5", + "lazy_static", + "num_cpus", +] + [[package]] name = "redox_syscall" version = "0.2.10" @@ -1968,7 +1994,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "assign", "js_int", @@ -1989,7 +2015,7 @@ dependencies = [ [[package]] name = "ruma-api" version = "0.18.3" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "bytes", "http", @@ -2005,7 +2031,7 @@ dependencies = [ [[package]] name = "ruma-api-macros" version = "0.18.3" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2016,7 +2042,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.4.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "ruma-api", "ruma-common", @@ -2030,7 +2056,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.12.2" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "assign", "bytes", @@ -2050,7 +2076,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.6.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "indexmap", "js_int", @@ -2065,7 +2091,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.24.5" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "indoc", "js_int", @@ -2081,7 +2107,7 @@ dependencies = [ [[package]] name = "ruma-events-macros" version = "0.24.5" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2092,7 +2118,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.3.1" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "js_int", "ruma-api", @@ -2107,7 +2133,7 @@ dependencies = [ [[package]] name = "ruma-identifiers" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "paste", "rand 0.8.4", @@ -2121,7 +2147,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-macros" version = "0.20.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "quote", "ruma-identifiers-validation", @@ -2131,12 +2157,12 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" [[package]] name = "ruma-identity-service-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "js_int", "ruma-api", @@ -2149,7 +2175,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.3.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "js_int", "ruma-api", @@ -2164,7 +2190,7 @@ dependencies = [ [[package]] name = "ruma-serde" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "bytes", "form_urlencoded", @@ -2178,7 +2204,7 @@ dependencies = [ [[package]] name = "ruma-serde-macros" version = "0.5.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "proc-macro-crate", "proc-macro2", @@ -2189,7 +2215,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.9.0" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "base64 0.13.0", "ed25519-dalek", @@ -2206,7 +2232,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.4.1" -source = "git+https://github.com/ruma/ruma?rev=a6a1224652912a957b09f136ec5da2686be6e0e2#a6a1224652912a957b09f136ec5da2686be6e0e2" +source = "git+https://github.com/timokoesters/ruma?rev=6e9823de284967a41100db0e3134319f4b6a0cfa#6e9823de284967a41100db0e3134319f4b6a0cfa" dependencies = [ "itertools 0.10.1", "js_int", diff --git a/Cargo.toml b/Cargo.toml index 593a1fd..40ac1b1 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -19,8 +19,8 @@ rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle request # Used for matrix spec type definitions and helpers #ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -ruma = { git = "https://github.com/ruma/ruma", rev = "a6a1224652912a957b09f136ec5da2686be6e0e2", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } -#ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +#ruma = { git = "https://github.com/ruma/ruma", rev = "a6a1224652912a957b09f136ec5da2686be6e0e2", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } +ruma = { git = "https://github.com/timokoesters/ruma", rev = "6e9823de284967a41100db0e3134319f4b6a0cfa", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] } # Used for long polling and federation sender, should be the same as rocket::tokio @@ -79,6 +79,7 @@ num_cpus = "1.13.0" threadpool = "1.8.1" heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true } thread_local = "1.1.3" +rayon = "1.5.1" [features] default = ["conduit_bin", "backend_sqlite"] diff --git a/src/client_server/account.rs b/src/client_server/account.rs index fb33842..636c452 100644 --- a/src/client_server/account.rs +++ b/src/client_server/account.rs @@ -28,6 +28,7 @@ use ruma::{ identifiers::RoomName, push, RoomAliasId, RoomId, RoomVersionId, UserId, }; +use serde_json::value::RawValue; use tracing::info; use register::RegistrationKind; @@ -279,7 +280,10 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&content).expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -294,16 +298,19 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(conduit_user.to_string()), redacts: None, @@ -322,13 +329,16 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, - content: serde_json::to_value( - ruma::events::room::power_levels::PowerLevelsEventContent { - users, - ..Default::default() - }, + content: RawValue::from_string( + serde_json::to_string( + &ruma::events::room::power_levels::PowerLevelsEventContent { + users, + ..Default::default() + }, + ) + .expect("event is valid, we just created it"), ) - .expect("event is valid, we just created it"), + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -343,10 +353,13 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomJoinRules, - content: serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&join_rules::JoinRulesEventContent::new( + join_rules::JoinRule::Invite, + )) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -361,12 +374,13 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value( - history_visibility::HistoryVisibilityEventContent::new( + content: RawValue::from_string( + serde_json::to_string(&history_visibility::HistoryVisibilityEventContent::new( history_visibility::HistoryVisibility::Shared, - ), + )) + .expect("event is valid, we just created it"), ) - .expect("event is valid, we just created it"), + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -381,10 +395,13 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomGuestAccess, - content: serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, - )) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&guest_access::GuestAccessEventContent::new( + guest_access::GuestAccess::Forbidden, + )) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -402,8 +419,11 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, - content: serde_json::to_value(name::NameEventContent::new(Some(room_name))) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&name::NameEventContent::new(Some(room_name))) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -417,10 +437,13 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { - topic: format!("Manage {}", db.globals.server_name()), - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&topic::TopicEventContent { + topic: format!("Manage {}", db.globals.server_name()), + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -439,11 +462,14 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCanonicalAlias, - content: serde_json::to_value(canonical_alias::CanonicalAliasEventContent { - alias: Some(alias.clone()), - alt_aliases: Vec::new(), - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&canonical_alias::CanonicalAliasEventContent { + alias: Some(alias.clone()), + alt_aliases: Vec::new(), + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -460,16 +486,19 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: None, - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: None, + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(user_id.to_string()), redacts: None, @@ -482,16 +511,19 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: Some(displayname), - avatar_url: None, - is_direct: None, - third_party_invite: None, - blurhash: None, - reason: None, - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: Some(displayname), + avatar_url: None, + is_direct: None, + third_party_invite: None, + blurhash: None, + reason: None, + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(user_id.to_string()), redacts: None, @@ -506,11 +538,11 @@ pub async fn register_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMessage, - content: serde_json::to_value(message::MessageEventContent::text_html( + content: RawValue::from_string(serde_json::to_string(&message::MessageEventContent::text_html( "## Thank you for trying out Conduit!\n\nConduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Website: https://conduit.rs\n> Git and Documentation: https://gitlab.com/famedly/conduit\n> Report issues: https://gitlab.com/famedly/conduit/-/issues\n\nHere are some rooms you can join (by typing the command):\n\nConduit room (Ask questions and get notified on updates):\n`/join #conduit:fachschaften.org`\n\nConduit lounge (Off-topic, only Conduit users are allowed to join)\n`/join #conduit-lounge:conduit.rs`".to_owned(), "

Thank you for trying out Conduit!

\n

Conduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.

\n

Helpful links:

\n
\n

Website: https://conduit.rs
Git and Documentation: https://gitlab.com/famedly/conduit
Report issues: https://gitlab.com/famedly/conduit/-/issues

\n
\n

Here are some rooms you can join (by typing the command):

\n

Conduit room (Ask questions and get notified on updates):
/join #conduit:fachschaften.org

\n

Conduit lounge (Off-topic, only Conduit users are allowed to join)
/join #conduit-lounge:conduit.rs

\n".to_owned(), )) - .expect("event is valid, we just created it"), + .expect("event is valid, we just created it")).expect("string is valid"), unsigned: None, state_key: None, redacts: None, @@ -721,7 +753,10 @@ pub async fn deactivate_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&event).expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, diff --git a/src/client_server/directory.rs b/src/client_server/directory.rs index 0065e51..180baf0 100644 --- a/src/client_server/directory.rs +++ b/src/client_server/directory.rs @@ -20,7 +20,6 @@ use ruma::{ room::{avatar, canonical_alias, guest_access, history_visibility, name, topic}, EventType, }, - serde::Raw, ServerName, UInt, }; use tracing::{info, warn}; @@ -217,76 +216,68 @@ pub(crate) async fn get_public_rooms_filtered_helper( } } - let mut all_rooms = - db.rooms - .public_rooms() - .map(|room_id| { - let room_id = room_id?; + let mut all_rooms = db + .rooms + .public_rooms() + .map(|room_id| { + let room_id = room_id?; - let chunk = PublicRoomsChunk { - aliases: Vec::new(), - canonical_alias: db - .rooms - .room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok(serde_json::from_value::< - Raw, - >(s.content.clone()) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| { - Error::bad_database("Invalid canonical alias event in database.") - })? - .alias) - })?, - name: db - .rooms - .room_state_get(&room_id, &EventType::RoomName, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok(serde_json::from_value::>( - s.content.clone(), + let chunk = PublicRoomsChunk { + aliases: Vec::new(), + canonical_alias: db + .rooms + .room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")? + .map_or(Ok::<_, Error>(None), |s| { + Ok( + serde_json::from_str::( + s.content.get(), ) - .expect("from_value::> can never fail") - .deserialize() .map_err(|_| { - Error::bad_database("Invalid room name event in database.") + Error::bad_database("Invalid canonical alias event in database.") })? - .name) - })?, - num_joined_members: db - .rooms - .room_joined_count(&room_id)? - .unwrap_or_else(|| { - warn!("Room {} has no member count", room_id); - 0 - }) - .try_into() - .expect("user count should not be that big"), - topic: db - .rooms - .room_state_get(&room_id, &EventType::RoomTopic, "")? - .map_or(Ok::<_, Error>(None), |s| { - Ok(Some( - serde_json::from_value::>( - s.content.clone(), - ) - .expect("from_value::> can never fail") - .deserialize() + .alias, + ) + })?, + name: db + .rooms + .room_state_get(&room_id, &EventType::RoomName, "")? + .map_or(Ok::<_, Error>(None), |s| { + Ok( + serde_json::from_str::(s.content.get()) + .map_err(|_| { + Error::bad_database("Invalid room name event in database.") + })? + .name, + ) + })?, + num_joined_members: db + .rooms + .room_joined_count(&room_id)? + .unwrap_or_else(|| { + warn!("Room {} has no member count", room_id); + 0 + }) + .try_into() + .expect("user count should not be that big"), + topic: db + .rooms + .room_state_get(&room_id, &EventType::RoomTopic, "")? + .map_or(Ok::<_, Error>(None), |s| { + Ok(Some( + serde_json::from_str::(s.content.get()) .map_err(|_| { Error::bad_database("Invalid room topic event in database.") })? .topic, - )) - })?, - world_readable: db - .rooms - .room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")? - .map_or(Ok::<_, Error>(false), |s| { - Ok(serde_json::from_value::< - Raw, - >(s.content.clone()) - .expect("from_value::> can never fail") - .deserialize() + )) + })?, + world_readable: db + .rooms + .room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")? + .map_or(Ok::<_, Error>(false), |s| { + Ok(serde_json::from_str::< + history_visibility::HistoryVisibilityEventContent, + >(s.content.get()) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", @@ -294,80 +285,74 @@ pub(crate) async fn get_public_rooms_filtered_helper( })? .history_visibility == history_visibility::HistoryVisibility::WorldReadable) - })?, - guest_can_join: db - .rooms - .room_state_get(&room_id, &EventType::RoomGuestAccess, "")? - .map_or(Ok::<_, Error>(false), |s| { - Ok( - serde_json::from_value::>( - s.content.clone(), + })?, + guest_can_join: db + .rooms + .room_state_get(&room_id, &EventType::RoomGuestAccess, "")? + .map_or(Ok::<_, Error>(false), |s| { + Ok( + serde_json::from_str::( + s.content.get(), ) - .expect("from_value::> can never fail") - .deserialize() .map_err(|_| { Error::bad_database("Invalid room guest access event in database.") })? .guest_access == guest_access::GuestAccess::CanJoin, ) - })?, - avatar_url: db - .rooms - .room_state_get(&room_id, &EventType::RoomAvatar, "")? - .map(|s| { - Ok::<_, Error>( - serde_json::from_value::>( - s.content.clone(), - ) - .expect("from_value::> can never fail") - .deserialize() + })?, + avatar_url: db + .rooms + .room_state_get(&room_id, &EventType::RoomAvatar, "")? + .map(|s| { + Ok::<_, Error>( + serde_json::from_str::(s.content.get()) .map_err(|_| { Error::bad_database("Invalid room avatar event in database.") })? .url, - ) - }) - .transpose()? - // url is now an Option so we must flatten - .flatten(), - room_id, - }; - Ok(chunk) - }) - .filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms - .filter(|chunk| { - if let Some(query) = filter - .generic_search_term - .as_ref() - .map(|q| q.to_lowercase()) - { - if let Some(name) = &chunk.name { - if name.as_str().to_lowercase().contains(&query) { - return true; - } + ) + }) + .transpose()? + // url is now an Option so we must flatten + .flatten(), + room_id, + }; + Ok(chunk) + }) + .filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms + .filter(|chunk| { + if let Some(query) = filter + .generic_search_term + .as_ref() + .map(|q| q.to_lowercase()) + { + if let Some(name) = &chunk.name { + if name.as_str().to_lowercase().contains(&query) { + return true; } + } - if let Some(topic) = &chunk.topic { - if topic.to_lowercase().contains(&query) { - return true; - } + if let Some(topic) = &chunk.topic { + if topic.to_lowercase().contains(&query) { + return true; } + } - if let Some(canonical_alias) = &chunk.canonical_alias { - if canonical_alias.as_str().to_lowercase().contains(&query) { - return true; - } + if let Some(canonical_alias) = &chunk.canonical_alias { + if canonical_alias.as_str().to_lowercase().contains(&query) { + return true; } - - false - } else { - // No search term - true } - }) - // We need to collect all, so we can sort by member count - .collect::>(); + + false + } else { + // No search term + true + } + }) + // We need to collect all, so we can sort by member count + .collect::>(); all_rooms.sort_by(|l, r| r.num_joined_members.cmp(&l.num_joined_members)); diff --git a/src/client_server/membership.rs b/src/client_server/membership.rs index 146af79..6b4868c 100644 --- a/src/client_server/membership.rs +++ b/src/client_server/membership.rs @@ -5,6 +5,7 @@ use crate::{ server_server, utils, ConduitResult, Database, Error, Result, Ruma, }; use member::{MemberEventContent, MembershipState}; +use rayon::prelude::*; use ruma::{ api::{ client::{ @@ -15,7 +16,10 @@ use ruma::{ unban_user, IncomingThirdPartySigned, }, }, - federation::{self, membership::create_invite}, + federation::{ + self, + membership::{create_invite, create_join_event}, + }, }, events::{ pdu::Pdu, @@ -26,13 +30,13 @@ use ruma::{ state_res::{self, RoomVersion}, uint, EventId, RoomId, RoomVersionId, ServerName, UserId, }; +use serde_json::value::RawValue; use std::{ - collections::{hash_map::Entry, BTreeMap, HashMap, HashSet}, + collections::{BTreeMap, HashMap, HashSet}, convert::{TryFrom, TryInto}, sync::{Arc, RwLock}, - time::{Duration, Instant}, }; -use tracing::{debug, error, warn}; +use tracing::{error, warn}; #[cfg(feature = "conduit_bin")] use rocket::{get, post}; @@ -204,7 +208,7 @@ pub async fn kick_user_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mut event = serde_json::from_value::>( + let mut event = serde_json::from_str::>( db.rooms .room_state_get( &body.room_id, @@ -216,7 +220,7 @@ pub async fn kick_user_route( "Cannot kick member that's not in the room.", ))? .content - .clone(), + .get(), ) .expect("Raw::from_value always works") .deserialize() @@ -238,7 +242,10 @@ pub async fn kick_user_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&event).expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, @@ -290,12 +297,9 @@ pub async fn ban_user_route( reason: None, }), |event| { - let mut event = serde_json::from_value::>( - event.content.clone(), - ) - .expect("Raw::from_value always works") - .deserialize() - .map_err(|_| Error::bad_database("Invalid member event in database."))?; + let mut event = + serde_json::from_str::(event.content.get()) + .map_err(|_| Error::bad_database("Invalid member event in database."))?; event.membership = ruma::events::room::member::MembershipState::Ban; Ok(event) }, @@ -314,7 +318,10 @@ pub async fn ban_user_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&event).expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, @@ -346,7 +353,7 @@ pub async fn unban_user_route( ) -> ConduitResult { let sender_user = body.sender_user.as_ref().expect("user is authenticated"); - let mut event = serde_json::from_value::>( + let mut event = serde_json::from_str::( db.rooms .room_state_get( &body.room_id, @@ -358,10 +365,8 @@ pub async fn unban_user_route( "Cannot unban a user who is not banned.", ))? .content - .clone(), + .get(), ) - .expect("from_value::> can never fail") - .deserialize() .map_err(|_| Error::bad_database("Invalid member event in database."))?; event.membership = ruma::events::room::member::MembershipState::Leave; @@ -379,7 +384,10 @@ pub async fn unban_user_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&event).expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(body.user_id.to_string()), redacts: None, @@ -563,6 +571,7 @@ async fn join_room_by_id_helper( }, ) .await; + warn!("Make join done"); make_join_response_and_server = make_join_response.map(|r| (r, remote_server)); @@ -658,93 +667,193 @@ async fn join_room_by_id_helper( ) .await?; + warn!("Send join done"); + db.rooms.get_or_create_shortroomid(room_id, &db.globals)?; let pdu = PduEvent::from_id_val(&event_id, join_event.clone()) .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?; - let mut state = HashMap::new(); - let pub_key_map = RwLock::new(BTreeMap::new()); + let pub_key_map = Arc::new(RwLock::new(BTreeMap::new())); + let missing_servers = Arc::new(RwLock::new(BTreeMap::new())); - server_server::fetch_join_signing_keys( - &send_join_response, - &room_version, - &pub_key_map, - db, - ) - .await?; - - for result in send_join_response - .room_state - .state - .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, db)) - { - let (event_id, value) = match result { - Ok(t) => t, - Err(_) => continue, - }; - - let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| { - warn!("{:?}: {}", value, e); - Error::BadServerResponse("Invalid PDU in send_join response.") - })?; - - db.rooms.add_pdu_outlier(&event_id, &value)?; - if let Some(state_key) = &pdu.state_key { - let shortstatekey = - db.rooms - .get_or_create_shortstatekey(&pdu.kind, state_key, &db.globals)?; - state.insert(shortstatekey, pdu.event_id.clone()); - } - } - - let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey( - &pdu.kind, - pdu.state_key - .as_ref() - .expect("Pdu is a membership state event"), - &db.globals, - )?; - - state.insert(incoming_shortstatekey, pdu.event_id.clone()); + let create_join_event::RoomState { + state: mut room_state_state, + auth_chain: mut room_state_auth_chain, + } = send_join_response.room_state; let create_shortstatekey = db .rooms .get_shortstatekey(&EventType::RoomCreate, "")? .expect("Room exists"); - if state.get(&create_shortstatekey).is_none() { + let mut saw_create_event = false; + + warn!("Parsing send join response state"); + const CHUNK_SIZE: usize = 500; + let mut parsed_state = room_state_state + .par_chunks_mut(CHUNK_SIZE) + .filter_map(|pdus| { + let mut r = HashMap::with_capacity(CHUNK_SIZE); + for pdu in pdus { + let (id, value) = get_event_id(&pdu, &room_version).ok()?; + r.insert(id, value); + } + + let mut missing_servers = missing_servers.write().unwrap(); + let mut pub_key_map = pub_key_map.write().unwrap(); + for (_, value) in &r { + server_server::get_server_keys_from_cache( + &value, + &mut missing_servers, + &mut pub_key_map, + &db, + ) + .ok()?; + } + + Some(r) + }) + .collect::>(); + + warn!("Parsing send join response auth chain"); + let mut parsed_chain = room_state_auth_chain + .par_chunks_mut(CHUNK_SIZE) + .filter_map(|pdus| { + let mut r = HashMap::with_capacity(CHUNK_SIZE); + for pdu in pdus { + let (id, value) = get_event_id(&pdu, &room_version).ok()?; + r.insert(id, value); + } + + let mut missing_servers = missing_servers.write().unwrap(); + let mut pub_key_map = pub_key_map.write().unwrap(); + for (_, value) in &r { + server_server::get_server_keys_from_cache( + &value, + &mut missing_servers, + &mut pub_key_map, + &db, + ) + .ok()?; + } + + Some(r) + }) + .collect::>(); + + warn!("Fetching send join signing keys"); + server_server::fetch_join_signing_keys(missing_servers, &pub_key_map, db).await?; + + warn!("Validating state"); + parsed_state.par_iter_mut().for_each(|chunk| { + let mut bad_events = Vec::new(); + for (event_id, value) in chunk.iter_mut() { + if let Err(e) = ruma::signatures::verify_event( + &*pub_key_map.read().unwrap(), + &value, + &room_version, + ) { + warn!("Event {} failed verification {:?} {}", event_id, value, e); + bad_events.push(event_id.clone()); + continue; + } + + value.insert( + "event_id".to_owned(), + CanonicalJsonValue::String(event_id.as_str().to_owned()), + ); + } + for id in bad_events { + chunk.remove(&id); + } + }); + + warn!("Inserting state"); + db.rooms + .add_pdu_outlier_batch(&mut parsed_state.iter().flatten())?; + + warn!("Compressing state"); + let state = parsed_state + .iter() + .flatten() + .map(|(event_id, value)| { + let kind = if let Some(s) = value.get("type").and_then(|s| s.as_str()) { + s + } else { + warn!("Event {} has no type: {:?}", event_id, value); + return Ok(None); + }; + + if let Some(state_key) = value.get("state_key").and_then(|s| s.as_str()) { + let shortstatekey = db.rooms.get_or_create_shortstatekey( + &EventType::from(kind), + state_key, + &db.globals, + )?; + + if shortstatekey == create_shortstatekey { + saw_create_event = true; + } + + Ok(Some(db.rooms.compress_state_event( + shortstatekey, + &event_id, + &db.globals, + )?)) + } else { + Ok(None) + } + }) + .filter_map(|r| r.transpose()) + .collect::>>()?; + + if !saw_create_event { return Err(Error::BadServerResponse("State contained no create event.")); } - db.rooms.force_state( + warn!("Validating chain"); + parsed_chain.par_iter_mut().for_each(|chunk| { + let mut bad_events = Vec::new(); + for (event_id, value) in chunk.iter_mut() { + if let Err(e) = ruma::signatures::verify_event( + &*pub_key_map.read().unwrap(), + &value, + &room_version, + ) { + warn!("Event {} failed verification {:?} {}", event_id, value, e); + bad_events.push(event_id.clone()); + continue; + } + + value.insert( + "event_id".to_owned(), + CanonicalJsonValue::String(event_id.as_str().to_owned()), + ); + } + for id in bad_events { + chunk.remove(&id); + } + }); + + warn!("Inserting chain"); + db.rooms + .add_pdu_outlier_batch(&mut parsed_chain.iter().flatten())?; + + warn!("Forcing state of room"); + db.rooms.force_state_new( room_id, - state - .into_iter() - .map(|(k, id)| db.rooms.compress_state_event(k, &id, &db.globals)) - .collect::>>()?, + state, + &mut parsed_state.iter().flat_map(|m| m.values()), db, )?; - for result in send_join_response - .room_state - .auth_chain - .iter() - .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, db)) - { - let (event_id, value) = match result { - Ok(t) => t, - Err(_) => continue, - }; - - db.rooms.add_pdu_outlier(&event_id, &value)?; - } - // We append to state before appending the pdu, so we don't have a moment in time with the // pdu without it's state. This is okay because append_pdu can't fail. + warn!("Appending join event to state"); let statehashid = db.rooms.append_to_state(&pdu, &db.globals)?; + warn!("Adding join event to db"); db.rooms.append_pdu( &pdu, utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"), @@ -752,6 +861,7 @@ async fn join_room_by_id_helper( db, )?; + warn!("Updating room state to join event"); // We set the room state after inserting the pdu, so that we never have a moment in time // where events in the current room state do not exist db.rooms.set_room_state(room_id, statehashid)?; @@ -769,7 +879,10 @@ async fn join_room_by_id_helper( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event).expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&event).expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, @@ -788,16 +901,15 @@ async fn join_room_by_id_helper( Ok(join_room_by_id::Response::new(room_id.clone()).into()) } -fn validate_and_add_event_id( +fn get_event_id( pdu: &Raw, room_version: &RoomVersionId, - pub_key_map: &RwLock>>, - db: &Database, ) -> Result<(EventId, CanonicalJsonObject)> { - let mut value = serde_json::from_str::(pdu.json().get()).map_err(|e| { - error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); + let value = serde_json::from_str::(pdu.json().get()).map_err(|e| { + warn!("Invalid PDU in server response: {:?}: {:?}", pdu, e); Error::BadServerResponse("Invalid PDU in server response") })?; + let event_id = EventId::try_from(&*format!( "${}", ruma::signatures::reference_hash(&value, room_version) @@ -805,49 +917,6 @@ fn validate_and_add_event_id( )) .expect("ruma's reference hashes are valid event ids"); - let back_off = |id| match db.globals.bad_event_ratelimiter.write().unwrap().entry(id) { - Entry::Vacant(e) => { - e.insert((Instant::now(), 1)); - } - Entry::Occupied(mut e) => *e.get_mut() = (Instant::now(), e.get().1 + 1), - }; - - if let Some((time, tries)) = db - .globals - .bad_event_ratelimiter - .read() - .unwrap() - .get(&event_id) - { - // Exponential backoff - let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries); - if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { - min_elapsed_duration = Duration::from_secs(60 * 60 * 24); - } - - if time.elapsed() < min_elapsed_duration { - debug!("Backing off from {}", event_id); - return Err(Error::BadServerResponse("bad event, still backing off")); - } - } - - if let Err(e) = ruma::signatures::verify_event( - &*pub_key_map - .read() - .map_err(|_| Error::bad_database("RwLock is poisoned."))?, - &value, - room_version, - ) { - warn!("Event {} failed verification {:?} {}", event_id, pdu, e); - back_off(event_id); - return Err(Error::BadServerResponse("Event failed verification.")); - } - - value.insert( - "event_id".to_owned(), - CanonicalJsonValue::String(event_id.as_str().to_owned()), - ); - Ok((event_id, value)) } @@ -884,7 +953,7 @@ pub(crate) async fn invite_helper<'a>( let create_event_content = create_event .as_ref() .map(|create_event| { - serde_json::from_value::>(create_event.content.clone()) + serde_json::from_str::>(create_event.content.get()) .expect("Raw::from_value always works.") .deserialize() .map_err(|e| { @@ -910,16 +979,19 @@ pub(crate) async fn invite_helper<'a>( let room_version = RoomVersion::new(&room_version_id).expect("room version is supported"); - let content = serde_json::to_value(MemberEventContent { - avatar_url: None, - displayname: None, - is_direct: Some(is_direct), - membership: MembershipState::Invite, - third_party_invite: None, - blurhash: None, - reason: None, - }) - .expect("member event is valid value"); + let content = RawValue::from_string( + serde_json::to_string(&MemberEventContent { + avatar_url: None, + displayname: None, + is_direct: Some(is_direct), + membership: MembershipState::Invite, + third_party_invite: None, + blurhash: None, + reason: None, + }) + .expect("member event is valid value"), + ) + .expect("string is valid"); let state_key = user_id.to_string(); let kind = EventType::RoomMember; @@ -946,7 +1018,7 @@ pub(crate) async fn invite_helper<'a>( unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); unsigned.insert( "prev_sender".to_owned(), - serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), + serde_json::from_str(prev_pdu.sender.as_str()).expect("UserId is valid string"), ); } @@ -959,6 +1031,7 @@ pub(crate) async fn invite_helper<'a>( .expect("time is valid"), kind, content, + parsed_content: RwLock::new(None), state_key: Some(state_key), prev_events, depth, @@ -967,11 +1040,20 @@ pub(crate) async fn invite_helper<'a>( .map(|(_, pdu)| pdu.event_id.clone()) .collect(), redacts: None, - unsigned, + unsigned: if unsigned.is_empty() { + None + } else { + Some( + RawValue::from_string( + serde_json::to_string(&unsigned).expect("to_string always works"), + ) + .expect("string is valid"), + ) + }, hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), }, - signatures: BTreeMap::new(), + signatures: None, }; let auth_check = state_res::auth_check( @@ -1116,16 +1198,19 @@ pub(crate) async fn invite_helper<'a>( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Invite, - displayname: db.users.displayname(user_id)?, - avatar_url: db.users.avatar_url(user_id)?, - is_direct: Some(is_direct), - third_party_invite: None, - blurhash: db.users.blurhash(user_id)?, - reason: None, - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&member::MemberEventContent { + membership: member::MembershipState::Invite, + displayname: db.users.displayname(user_id)?, + avatar_url: db.users.avatar_url(user_id)?, + is_direct: Some(is_direct), + third_party_invite: None, + blurhash: db.users.blurhash(user_id)?, + reason: None, + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(user_id.to_string()), redacts: None, diff --git a/src/client_server/profile.rs b/src/client_server/profile.rs index ab7fb02..62c9e1b 100644 --- a/src/client_server/profile.rs +++ b/src/client_server/profile.rs @@ -10,8 +10,8 @@ use ruma::{ federation::{self, query::get_profile_information::v1::ProfileField}, }, events::EventType, - serde::Raw, }; +use serde_json::value::RawValue; use std::{convert::TryInto, sync::Arc}; #[cfg(feature = "conduit_bin")] @@ -45,29 +45,30 @@ pub async fn set_displayname_route( Ok::<_, Error>(( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - displayname: body.displayname.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_user.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( - "Tried to send displayname update for user not in the \ + content: RawValue::from_string( + serde_json::to_string(&ruma::events::room::member::MemberEventContent { + displayname: body.displayname.clone(), + ..serde_json::from_str( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_user.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( + "Tried to send displayname update for user not in the \ room.", - ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), + ) + })? + .content + .get(), + ) + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, @@ -190,29 +191,30 @@ pub async fn set_avatar_url_route( Ok::<_, Error>(( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(ruma::events::room::member::MemberEventContent { - avatar_url: body.avatar_url.clone(), - ..serde_json::from_value::>( - db.rooms - .room_state_get( - &room_id, - &EventType::RoomMember, - &sender_user.to_string(), - )? - .ok_or_else(|| { - Error::bad_database( - "Tried to send displayname update for user not in the \ + content: RawValue::from_string( + serde_json::to_string(&ruma::events::room::member::MemberEventContent { + avatar_url: body.avatar_url.clone(), + ..serde_json::from_str( + db.rooms + .room_state_get( + &room_id, + &EventType::RoomMember, + &sender_user.to_string(), + )? + .ok_or_else(|| { + Error::bad_database( + "Tried to send displayname update for user not in the \ room.", - ) - })? - .content - .clone(), - ) - .expect("from_value::> can never fail") - .deserialize() - .map_err(|_| Error::bad_database("Database contains invalid PDU."))? - }) - .expect("event is valid, we just created it"), + ) + })? + .content + .get(), + ) + .map_err(|_| Error::bad_database("Database contains invalid PDU."))? + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, diff --git a/src/client_server/redact.rs b/src/client_server/redact.rs index 4b5219b..8b23e64 100644 --- a/src/client_server/redact.rs +++ b/src/client_server/redact.rs @@ -8,6 +8,7 @@ use ruma::{ #[cfg(feature = "conduit_bin")] use rocket::put; +use serde_json::value::RawValue; /// # `PUT /_matrix/client/r0/rooms/{roomId}/redact/{eventId}/{txnId}` /// @@ -38,10 +39,13 @@ pub async fn redact_event_route( let event_id = db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomRedaction, - content: serde_json::to_value(redaction::RedactionEventContent { - reason: body.reason.clone(), - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&redaction::RedactionEventContent { + reason: body.reason.clone(), + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: None, redacts: Some(body.event_id.clone()), diff --git a/src/client_server/room.rs b/src/client_server/room.rs index 5a02699..07e669a 100644 --- a/src/client_server/room.rs +++ b/src/client_server/room.rs @@ -11,9 +11,9 @@ use ruma::{ room::{guest_access, history_visibility, join_rules, member, name, topic}, EventType, }, - serde::Raw, RoomAliasId, RoomId, RoomVersionId, }; +use serde_json::value::RawValue; use std::{cmp::max, collections::BTreeMap, convert::TryFrom, sync::Arc}; use tracing::{info, warn}; @@ -101,7 +101,10 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCreate, - content: serde_json::to_value(content).expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&content).expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -116,16 +119,19 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(sender_user)?, - avatar_url: db.users.avatar_url(sender_user)?, - is_direct: Some(body.is_direct), - third_party_invite: None, - blurhash: db.users.blurhash(sender_user)?, - reason: None, - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, + is_direct: Some(body.is_direct), + third_party_invite: None, + blurhash: db.users.blurhash(sender_user)?, + reason: None, + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, @@ -180,7 +186,8 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, - content: power_levels_content, + content: RawValue::from_string(power_levels_content.to_string()) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -196,13 +203,16 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCanonicalAlias, - content: serde_json::to_value( - ruma::events::room::canonical_alias::CanonicalAliasEventContent { - alias: Some(room_alias_id.clone()), - alt_aliases: vec![], - }, + content: RawValue::from_string( + serde_json::to_string( + &ruma::events::room::canonical_alias::CanonicalAliasEventContent { + alias: Some(room_alias_id.clone()), + alt_aliases: vec![], + }, + ) + .expect("We checked that alias earlier, it must be fine"), ) - .expect("We checked that alias earlier, it must be fine"), + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -220,17 +230,15 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomJoinRules, - content: match preset { - create_room::RoomPreset::PublicChat => serde_json::to_value( - join_rules::JoinRulesEventContent::new(join_rules::JoinRule::Public), - ) + content: RawValue::from_string( + serde_json::to_string(&join_rules::JoinRulesEventContent::new(match preset { + create_room::RoomPreset::PublicChat => join_rules::JoinRule::Public, + // according to spec "invite" is the default + _ => join_rules::JoinRule::Invite, + })) .expect("event is valid, we just created it"), - // according to spec "invite" is the default - _ => serde_json::to_value(join_rules::JoinRulesEventContent::new( - join_rules::JoinRule::Invite, - )) - .expect("event is valid, we just created it"), - }, + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -245,10 +253,13 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomHistoryVisibility, - content: serde_json::to_value(history_visibility::HistoryVisibilityEventContent::new( - history_visibility::HistoryVisibility::Shared, - )) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&history_visibility::HistoryVisibilityEventContent::new( + history_visibility::HistoryVisibility::Shared, + )) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -263,18 +274,14 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomGuestAccess, - content: match preset { - create_room::RoomPreset::PublicChat => { - serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::Forbidden, - )) - .expect("event is valid, we just created it") - } - _ => serde_json::to_value(guest_access::GuestAccessEventContent::new( - guest_access::GuestAccess::CanJoin, - )) + content: RawValue::from_string( + serde_json::to_string(&guest_access::GuestAccessEventContent::new(match preset { + create_room::RoomPreset::PublicChat => guest_access::GuestAccess::Forbidden, + _ => guest_access::GuestAccess::CanJoin, + })) .expect("event is valid, we just created it"), - }, + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -306,8 +313,11 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomName, - content: serde_json::to_value(name::NameEventContent::new(Some(name.clone()))) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&name::NameEventContent::new(Some(name.clone()))) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -323,10 +333,13 @@ pub async fn create_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomTopic, - content: serde_json::to_value(topic::TopicEventContent { - topic: topic.clone(), - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&topic::TopicEventContent { + topic: topic.clone(), + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -477,11 +490,14 @@ pub async fn upgrade_room_route( let tombstone_event_id = db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomTombstone, - content: serde_json::to_value(ruma::events::room::tombstone::TombstoneEventContent { - body: "This room has been replaced".to_string(), - replacement_room: replacement_room.clone(), - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&ruma::events::room::tombstone::TombstoneEventContent { + body: "This room has been replaced".to_string(), + replacement_room: replacement_room.clone(), + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -505,15 +521,13 @@ pub async fn upgrade_room_route( let state_lock = mutex_state.lock().await; // Get the old room federations status - let federate = serde_json::from_value::>( + let federate = serde_json::from_str::( db.rooms .room_state_get(&body.room_id, &EventType::RoomCreate, "")? .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? .content - .clone(), + .get(), ) - .expect("Raw::from_value always works") - .deserialize() .map_err(|_| Error::bad_database("Invalid room event in database."))? .federate; @@ -533,8 +547,11 @@ pub async fn upgrade_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomCreate, - content: serde_json::to_value(create_event_content) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&create_event_content) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, @@ -549,16 +566,19 @@ pub async fn upgrade_room_route( db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(member::MemberEventContent { - membership: member::MembershipState::Join, - displayname: db.users.displayname(sender_user)?, - avatar_url: db.users.avatar_url(sender_user)?, - is_direct: None, - third_party_invite: None, - blurhash: db.users.blurhash(sender_user)?, - reason: None, - }) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&member::MemberEventContent { + membership: member::MembershipState::Join, + displayname: db.users.displayname(sender_user)?, + avatar_url: db.users.avatar_url(sender_user)?, + is_direct: None, + third_party_invite: None, + blurhash: db.users.blurhash(sender_user)?, + reason: None, + }) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(sender_user.to_string()), redacts: None, @@ -585,7 +605,7 @@ pub async fn upgrade_room_route( // Replicate transferable state events to the new room for event_type in transferable_state_events { let event_content = match db.rooms.room_state_get(&body.room_id, &event_type, "")? { - Some(v) => v.content.clone(), + Some(v) => v.content, None => continue, // Skipping missing events. }; @@ -612,15 +632,13 @@ pub async fn upgrade_room_route( // Get the old room power levels let mut power_levels_event_content = - serde_json::from_value::>( + serde_json::from_str::( db.rooms .room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")? .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))? .content - .clone(), + .get(), ) - .expect("database contains invalid PDU") - .deserialize() .map_err(|_| Error::bad_database("Invalid room event in database."))?; // Setting events_default and invite to the greater of 50 and users_default + 1 @@ -635,8 +653,11 @@ pub async fn upgrade_room_route( let _ = db.rooms.build_and_append_pdu( PduBuilder { event_type: EventType::RoomPowerLevels, - content: serde_json::to_value(power_levels_event_content) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&power_levels_event_content) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some("".to_owned()), redacts: None, diff --git a/src/client_server/state.rs b/src/client_server/state.rs index 24cc2a1..9534bda 100644 --- a/src/client_server/state.rs +++ b/src/client_server/state.rs @@ -112,7 +112,7 @@ pub async fn get_state_events_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_value::(event.content.clone()) + serde_json::from_str::(event.content.get()) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", @@ -164,7 +164,7 @@ pub async fn get_state_events_for_key_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_value::(event.content.clone()) + serde_json::from_str::(event.content.get()) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", @@ -190,7 +190,7 @@ pub async fn get_state_events_for_key_route( ))?; Ok(get_state_events_for_key::Response { - content: serde_json::from_value(event.content.clone()) + content: serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content in database"))?, } .into()) @@ -220,7 +220,7 @@ pub async fn get_state_events_for_empty_key_route( db.rooms .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")? .map(|event| { - serde_json::from_value::(event.content.clone()) + serde_json::from_str::(event.content.get()) .map_err(|_| { Error::bad_database( "Invalid room history visibility event in database.", @@ -246,7 +246,7 @@ pub async fn get_state_events_for_empty_key_route( ))?; Ok(get_state_events_for_key::Response { - content: serde_json::from_value(event.content.clone()) + content: serde_json::from_str(event.content.get()) .map_err(|_| Error::bad_database("Invalid event content in database"))?, } .into()) diff --git a/src/client_server/sync.rs b/src/client_server/sync.rs index 2d5ad27..897c085 100644 --- a/src/client_server/sync.rs +++ b/src/client_server/sync.rs @@ -287,9 +287,9 @@ async fn sync_helper( .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus .filter(|(_, pdu)| pdu.kind == EventType::RoomMember) .map(|(_, pdu)| { - let content = serde_json::from_value::< + let content = serde_json::from_str::< ruma::events::room::member::MemberEventContent, - >(pdu.content.clone()) + >(pdu.content.get()) .map_err(|_| Error::bad_database("Invalid member event in database."))?; if let Some(state_key) = &pdu.state_key { @@ -371,11 +371,9 @@ async fn sync_helper( sender_user.as_str(), )? .and_then(|pdu| { - serde_json::from_value::>( - pdu.content.clone(), + serde_json::from_str::( + pdu.content.get(), ) - .expect("Raw::from_value always works") - .deserialize() .map_err(|_| Error::bad_database("Invalid PDU in database.")) .ok() }); @@ -432,11 +430,9 @@ async fn sync_helper( continue; } - let new_membership = serde_json::from_value::< - Raw, - >(state_event.content.clone()) - .expect("Raw::from_value always works") - .deserialize() + let new_membership = serde_json::from_str::< + ruma::events::room::member::MemberEventContent, + >(state_event.content.get()) .map_err(|_| Error::bad_database("Invalid PDU in database."))? .membership; diff --git a/src/database/admin.rs b/src/database/admin.rs index 424e674..cfa6d4c 100644 --- a/src/database/admin.rs +++ b/src/database/admin.rs @@ -9,6 +9,7 @@ use ruma::{ events::{room::message, EventType}, UserId, }; +use serde_json::value::RawValue; use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard}; use tracing::warn; @@ -66,8 +67,11 @@ impl Admin { .build_and_append_pdu( PduBuilder { event_type: EventType::RoomMessage, - content: serde_json::to_value(message) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&message) + .expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: None, redacts: None, diff --git a/src/database/pusher.rs b/src/database/pusher.rs index b19f339..c8288d4 100644 --- a/src/database/pusher.rs +++ b/src/database/pusher.rs @@ -9,8 +9,10 @@ use ruma::{ }, IncomingResponse, OutgoingRequest, SendAccessToken, }, - events::{room::power_levels::PowerLevelsEventContent, AnySyncRoomEvent, EventType}, - identifiers::RoomName, + events::{ + room::{name::NameEventContent, power_levels::PowerLevelsEventContent}, + AnySyncRoomEvent, EventType, + }, push::{Action, PushConditionRoomCtx, PushFormat, Ruleset, Tweak}, serde::Raw, uint, RoomId, UInt, UserId, @@ -181,7 +183,7 @@ pub async fn send_push_notice( .rooms .room_state_get(&pdu.room_id, &EventType::RoomPowerLevels, "")? .map(|ev| { - serde_json::from_value(ev.content.clone()) + serde_json::from_str(ev.content.get()) .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) }) .transpose()? @@ -318,16 +320,18 @@ async fn send_notice( let user_name = db.users.displayname(&event.sender)?; notifi.sender_display_name = user_name.as_deref(); - let room_name = db - .rooms - .room_state_get(&event.room_id, &EventType::RoomName, "")? - .map(|pdu| match pdu.content.get("name") { - Some(serde_json::Value::String(s)) => { - Some(Box::::try_from(&**s).expect("room name is valid")) - } - _ => None, - }) - .flatten(); + + let room_name = if let Some(room_name_pdu) = + db.rooms + .room_state_get(&event.room_id, &EventType::RoomName, "")? + { + serde_json::from_str::(room_name_pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid room name event in database."))? + .name + } else { + None + }; + notifi.room_name = room_name.as_deref(); send_request( diff --git a/src/database/rooms.rs b/src/database/rooms.rs index ec03e3a..0153d11 100644 --- a/src/database/rooms.rs +++ b/src/database/rooms.rs @@ -2,6 +2,7 @@ mod edus; pub use edus::RoomEdus; use member::MembershipState; +use serde_json::value::RawValue; use crate::{pdu::PduBuilder, server_server, utils, Database, Error, PduEvent, Result}; use lru_cache::LruCache; @@ -96,7 +97,7 @@ pub struct Rooms { /// RoomId + EventId -> Parent PDU EventId. pub(super) referencedevents: Arc, - pub(super) pdu_cache: Mutex>>, + pub(super) pdu_cache: Mutex>>, pub(super) shorteventid_cache: Mutex>>, pub(super) auth_chain_cache: Mutex, Arc>>>, pub(super) eventidshort_cache: Mutex>, @@ -137,7 +138,7 @@ impl Rooms { pub fn state_full( &self, shortstatehash: u64, - ) -> Result>> { + ) -> Result> { let full_state = self .load_shortstatehash_info(shortstatehash)? .pop() @@ -199,7 +200,7 @@ impl Rooms { shortstatehash: u64, event_type: &EventType, state_key: &str, - ) -> Result>> { + ) -> Result> { self.state_get_id(shortstatehash, event_type, state_key)? .map_or(Ok(None), |event_id| self.get_pdu(&event_id)) } @@ -243,8 +244,8 @@ impl Rooms { kind: &EventType, sender: &UserId, state_key: Option<&str>, - content: &serde_json::Value, - ) -> Result>> { + content: &serde_json::value::RawValue, + ) -> Result> { let shortstatehash = if let Some(current_shortstatehash) = self.current_shortstatehash(room_id)? { current_shortstatehash @@ -252,7 +253,12 @@ impl Rooms { return Ok(HashMap::new()); }; - let auth_events = state_res::auth_types_for_event(kind, sender, state_key, content); + let auth_events = state_res::auth_types_for_event( + kind, + sender, + state_key, + &serde_json::from_str(content.get()).expect("RawValue is valid json"), + ); let mut sauthevents = auth_events .into_iter() @@ -329,6 +335,75 @@ impl Rooms { .transpose() } + /// Force the creation of a new StateHash and insert it into the db. This version should be + /// used when creating new rooms. + /// + /// Whatever `state` is supplied to `force_state` becomes the new current room state snapshot. + #[tracing::instrument(skip(self, new_state_ids_compressed, state_values, db))] + pub fn force_state_new( + &self, + room_id: &RoomId, + new_state_ids_compressed: HashSet, + state_values: &mut dyn Iterator, + db: &Database, + ) -> Result<()> { + let state_hash = self.calculate_hash( + &new_state_ids_compressed + .iter() + .map(|bytes| &bytes[..]) + .collect::>(), + ); + + let (new_shortstatehash, _) = + self.get_or_create_shortstatehash(&state_hash, &db.globals)?; + + self.save_state_from_diff( + new_shortstatehash, + new_state_ids_compressed.clone(), + HashSet::new(), + 2, // every state change is 2 event changes on average + Vec::new(), + )?; + + for pdu in state_values { + if pdu.get("type").and_then(|val| val.as_str()) == Some("m.room.member") { + let content = pdu + .get("content") + .and_then(|o| o.as_object()) + .ok_or_else(|| Error::bad_database("Invalid content in pdu."))?; + if let Some(membership) = content.get("membership").and_then(|membership| { + serde_json::from_str::( + &serde_json::to_string(membership).expect("json is valid string"), + ) + .ok() + }) { + if let Some(state_key) = pdu + .get("state_key") + .and_then(|o| o.as_str()) + .and_then(|state_key| UserId::try_from(state_key).ok()) + { + if let Some(sender) = pdu + .get("sender") + .and_then(|o| o.as_str()) + .and_then(|state_key| UserId::try_from(state_key).ok()) + { + self.update_membership( + room_id, &state_key, membership, &sender, None, db, false, + )?; + } + } + } + } + } + + self.update_joined_count(room_id, db)?; + + self.roomid_shortstatehash + .insert(room_id.as_bytes(), &new_shortstatehash.to_be_bytes())?; + + Ok(()) + } + /// Force the creation of a new StateHash and insert it into the db. /// /// Whatever `state` is supplied to `force_state` becomes the new current room state snapshot. @@ -393,29 +468,28 @@ impl Rooms { }) { if let Some(pdu) = self.get_pdu_json(&event_id)? { if pdu.get("type").and_then(|val| val.as_str()) == Some("m.room.member") { - if let Ok(pdu) = serde_json::from_value::( - serde_json::to_value(&pdu).expect("CanonicalJsonObj is a valid JsonValue"), - ) { - if let Some(membership) = - pdu.content.get("membership").and_then(|membership| { - serde_json::from_value::( - membership.clone(), - ) - .ok() - }) + let content = pdu + .get("content") + .and_then(|o| o.as_object()) + .ok_or_else(|| Error::bad_database("Invalid content in pdu."))?; + if let Some(membership) = content.get("membership").and_then(|membership| { + serde_json::from_str::( + &serde_json::to_string(membership).expect("json is valid string"), + ) + .ok() + }) { + if let Some(state_key) = pdu + .get("state_key") + .and_then(|o| o.as_str()) + .and_then(|state_key| UserId::try_from(state_key).ok()) { - if let Some(state_key) = pdu - .state_key + if let Some(sender) = pdu + .get("sender") + .and_then(|o| o.as_str()) .and_then(|state_key| UserId::try_from(state_key).ok()) { self.update_membership( - room_id, - &state_key, - membership, - &pdu.sender, - None, - db, - false, + room_id, &state_key, membership, &sender, None, db, false, )?; } } @@ -915,7 +989,7 @@ impl Rooms { pub fn room_state_full( &self, room_id: &RoomId, - ) -> Result>> { + ) -> Result> { if let Some(current_shortstatehash) = self.current_shortstatehash(room_id)? { self.state_full(current_shortstatehash) } else { @@ -945,7 +1019,7 @@ impl Rooms { room_id: &RoomId, event_type: &EventType, state_key: &str, - ) -> Result>> { + ) -> Result> { if let Some(current_shortstatehash) = self.current_shortstatehash(room_id)? { self.state_get(current_shortstatehash, event_type, state_key) } else { @@ -1073,9 +1147,11 @@ impl Rooms { /// /// Checks the `eventid_outlierpdu` Tree if not found in the timeline. #[tracing::instrument(skip(self))] - pub fn get_pdu(&self, event_id: &EventId) -> Result>> { + pub fn get_pdu(&self, event_id: &EventId) -> Result> { if let Some(p) = self.pdu_cache.lock().unwrap().get_mut(event_id) { - return Ok(Some(Arc::clone(p))); + return Ok(Some( + serde_json::from_slice(p).expect("pdus in cache are valid"), + )); } if let Some(pdu) = self @@ -1092,18 +1168,13 @@ impl Rooms { })?)) }, )? - .map(|pdu| { - serde_json::from_slice(&pdu) - .map_err(|_| Error::bad_database("Invalid PDU in db.")) - .map(Arc::new) - }) - .transpose()? { - self.pdu_cache - .lock() - .unwrap() - .insert(event_id.clone(), Arc::clone(&pdu)); - Ok(Some(pdu)) + let parsed = + serde_json::from_slice(&pdu).map_err(|_| Error::bad_database("Invalid PDU in db.")); + + self.pdu_cache.lock().unwrap().insert(event_id.clone(), pdu); + + parsed } else { Ok(None) } @@ -1228,6 +1299,23 @@ impl Rooms { ) } + /// Append the PDU as an outlier. + /// + /// Any event given to this will be processed (state-res) on another thread. + #[tracing::instrument(skip(self, batch))] + pub fn add_pdu_outlier_batch( + &self, + batch: &mut dyn Iterator, + ) -> Result<()> { + self.eventid_outlierpdu + .insert_batch(&mut batch.map(|(event_id, pdu)| { + ( + event_id.as_bytes().to_vec(), + serde_json::to_vec(&pdu).expect("CanonicalJsonObject is valid"), + ) + })) + } + #[tracing::instrument(skip(self))] pub fn mark_event_soft_failed(&self, event_id: &EventId) -> Result<()> { self.softfailedeventids.insert(event_id.as_bytes(), &[]) @@ -1329,7 +1417,7 @@ impl Rooms { .rooms .room_state_get(&pdu.room_id, &EventType::RoomPowerLevels, "")? .map(|ev| { - serde_json::from_value(ev.content.clone()) + serde_json::from_str(ev.content.get()) .map_err(|_| Error::bad_database("invalid m.room.power_levels event")) }) .transpose()? @@ -1408,8 +1496,11 @@ impl Rooms { let target_user_id = UserId::try_from(state_key.clone()) .expect("This state_key was previously validated"); + let content = serde_json::from_str::(pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in pdu."))?; + let membership = serde_json::from_value::( - pdu.content + content .get("membership") .ok_or(Error::BadRequest( ErrorKind::InvalidParam, @@ -1447,7 +1538,10 @@ impl Rooms { } } EventType::RoomMessage => { - if let Some(body) = pdu.content.get("body").and_then(|b| b.as_str()) { + let content = serde_json::from_str::(pdu.content.get()) + .map_err(|_| Error::bad_database("Invalid content in pdu."))?; + + if let Some(body) = content.get("body").and_then(|b| b.as_str()) { let mut batch = body .split_terminator(|c: char| !c.is_alphanumeric()) .filter(|s| !s.is_empty()) @@ -1961,7 +2055,7 @@ impl Rooms { let create_event_content = create_event .as_ref() .map(|create_event| { - serde_json::from_value::>(create_event.content.clone()) + serde_json::from_str::>(create_event.content.get()) .expect("Raw::from_value always works.") .deserialize() .map_err(|e| { @@ -2000,7 +2094,10 @@ impl Rooms { let mut unsigned = unsigned.unwrap_or_default(); if let Some(state_key) = &state_key { if let Some(prev_pdu) = self.room_state_get(room_id, &event_type, state_key)? { - unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); + unsigned.insert( + "prev_content".to_owned(), + serde_json::from_str(prev_pdu.content.get()).expect("string is valid json"), + ); unsigned.insert( "prev_sender".to_owned(), serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), @@ -2017,6 +2114,7 @@ impl Rooms { .expect("time is valid"), kind: event_type, content, + parsed_content: RwLock::new(None), state_key, prev_events, depth, @@ -2025,11 +2123,20 @@ impl Rooms { .map(|(_, pdu)| pdu.event_id.clone()) .collect(), redacts, - unsigned, + unsigned: if unsigned.is_empty() { + None + } else { + Some( + RawValue::from_string( + serde_json::to_string(&unsigned).expect("to_string always works"), + ) + .expect("string is valid"), + ) + }, hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), }, - signatures: BTreeMap::new(), + signatures: None, }; let auth_check = state_res::auth_check( @@ -2205,7 +2312,7 @@ impl Rooms { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { - pdu.unsigned.remove("transaction_id"); + pdu.remove_transaction_id()?; } Ok((pdu_id, pdu)) })) @@ -2242,7 +2349,7 @@ impl Rooms { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { - pdu.unsigned.remove("transaction_id"); + pdu.remove_transaction_id()?; } Ok((pdu_id, pdu)) })) @@ -2279,7 +2386,7 @@ impl Rooms { let mut pdu = serde_json::from_slice::(&v) .map_err(|_| Error::bad_database("PDU in db is invalid."))?; if pdu.sender != user_id { - pdu.unsigned.remove("transaction_id"); + pdu.remove_transaction_id()?; } Ok((pdu_id, pdu)) })) @@ -2348,11 +2455,9 @@ impl Rooms { if let Some(predecessor) = self .room_state_get(room_id, &EventType::RoomCreate, "")? .and_then(|create| { - serde_json::from_value::< - Raw, - >(create.content.clone()) - .expect("Raw::from_value always works") - .deserialize() + serde_json::from_str::( + create.content.get(), + ) .ok() }) .and_then(|content| content.predecessor) @@ -2700,17 +2805,15 @@ impl Rooms { ); let state_lock = mutex_state.lock().await; - let mut event = serde_json::from_value::>( + let mut event = serde_json::from_str::( self.room_state_get(room_id, &EventType::RoomMember, &user_id.to_string())? .ok_or(Error::BadRequest( ErrorKind::BadState, "Cannot leave a room you are not a member of.", ))? .content - .clone(), + .get(), ) - .expect("from_value::> can never fail") - .deserialize() .map_err(|_| Error::bad_database("Invalid member event in database."))?; event.membership = member::MembershipState::Leave; @@ -2718,8 +2821,10 @@ impl Rooms { self.build_and_append_pdu( PduBuilder { event_type: EventType::RoomMember, - content: serde_json::to_value(event) - .expect("event is valid, we just created it"), + content: RawValue::from_string( + serde_json::to_string(&event).expect("event is valid, we just created it"), + ) + .expect("string is valid"), unsigned: None, state_key: Some(user_id.to_string()), redacts: None, diff --git a/src/database/sending.rs b/src/database/sending.rs index 70ff1b6..7e000e7 100644 --- a/src/database/sending.rs +++ b/src/database/sending.rs @@ -573,8 +573,14 @@ impl Sending { for pdu in pdus { // Redacted events are not notification targets (we don't send push for them) - if pdu.unsigned.get("redacted_because").is_some() { - continue; + if let Some(unsigned) = &pdu.unsigned { + if let Ok(unsigned) = + serde_json::from_str::(unsigned.get()) + { + if unsigned.get("redacted_because").is_some() { + continue; + } + } } let userid = diff --git a/src/pdu.rs b/src/pdu.rs index 8623b1a..ae08b8b 100644 --- a/src/pdu.rs +++ b/src/pdu.rs @@ -6,15 +6,15 @@ use ruma::{ AnySyncStateEvent, EventType, StateEvent, }, serde::{CanonicalJsonObject, CanonicalJsonValue, Raw}, - state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName, - ServerSigningKeyId, UInt, UserId, + state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName, UInt, + UserId, }; use serde::{Deserialize, Serialize}; -use serde_json::json; -use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom}; +use serde_json::{json, value::RawValue}; +use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom, sync::RwLock}; use tracing::warn; -#[derive(Clone, Deserialize, Serialize, Debug)] +#[derive(Deserialize, Serialize, Debug)] pub struct PduEvent { pub event_id: EventId, pub room_id: RoomId, @@ -22,7 +22,9 @@ pub struct PduEvent { pub origin_server_ts: UInt, #[serde(rename = "type")] pub kind: EventType, - pub content: serde_json::Value, + pub content: Box, + #[serde(skip)] + pub parsed_content: RwLock>, #[serde(skip_serializing_if = "Option::is_none")] pub state_key: Option, pub prev_events: Vec, @@ -30,16 +32,17 @@ pub struct PduEvent { pub auth_events: Vec, #[serde(skip_serializing_if = "Option::is_none")] pub redacts: Option, - #[serde(default, skip_serializing_if = "BTreeMap::is_empty")] - pub unsigned: BTreeMap, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub unsigned: Option>, pub hashes: EventHash, - pub signatures: BTreeMap, BTreeMap>, + #[serde(default, skip_serializing_if = "Option::is_none")] + pub signatures: Option>, // BTreeMap, BTreeMap> } impl PduEvent { #[tracing::instrument(skip(self))] pub fn redact(&mut self, reason: &PduEvent) -> crate::Result<()> { - self.unsigned.clear(); + self.unsigned = None; let allowed: &[&str] = match self.kind { EventType::RoomMember => &["membership"], @@ -59,10 +62,9 @@ impl PduEvent { _ => &[], }; - let old_content = self - .content - .as_object_mut() - .ok_or_else(|| Error::bad_database("PDU in db has invalid content."))?; + let mut old_content = + serde_json::from_str::>(self.content.get()) + .map_err(|_| Error::bad_database("PDU in db has invalid content."))?; let mut new_content = serde_json::Map::new(); @@ -72,12 +74,29 @@ impl PduEvent { } } - self.unsigned.insert( - "redacted_because".to_owned(), - serde_json::to_value(reason).expect("to_value(PduEvent) always works"), - ); + self.unsigned = Some(RawValue::from_string(serde_json::to_string(&json!({ + "redacted_because": serde_json::to_value(reason).expect("to_value(PduEvent) always works") + })).expect("to string always works")).expect("string is valid")); + + self.content = RawValue::from_string( + serde_json::to_string(&new_content).expect("to string always works"), + ) + .expect("string is valid"); + + Ok(()) + } - self.content = new_content.into(); + pub fn remove_transaction_id(&mut self) -> crate::Result<()> { + if let Some(unsigned) = &self.unsigned { + let mut unsigned = + serde_json::from_str::>>(unsigned.get()) + .map_err(|_| Error::bad_database("Invalid unsigned in pdu event"))?; + unsigned.remove("transaction_id"); + self.unsigned = Some( + RawValue::from_string(serde_json::to_string(&unsigned).expect("unsigned is valid")) + .expect("string is valid"), + ); + } Ok(()) } @@ -265,8 +284,14 @@ impl state_res::Event for PduEvent { &self.kind } - fn content(&self) -> &serde_json::Value { - &self.content + fn content(&self) -> serde_json::Value { + self.parsed_content + .write() + .unwrap() + .get_or_insert_with(|| { + serde_json::to_value(&self.content).expect("content is valid json") + }) + .clone() } fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch { @@ -298,11 +323,17 @@ impl state_res::Event for PduEvent { } fn signatures(&self) -> BTreeMap, BTreeMap> { - self.signatures.clone() + self.signatures + .as_ref() + .map(|raw| serde_json::from_str(raw.get()).expect("string is valid signatures json")) + .unwrap_or_default() } - fn unsigned(&self) -> &BTreeMap { - &self.unsigned + fn unsigned(&self) -> BTreeMap { + self.unsigned + .as_ref() + .map(|raw| serde_json::from_str(raw.get()).expect("string is valid json")) + .unwrap_or_default() } } @@ -352,7 +383,7 @@ pub(crate) fn gen_event_id_canonical_json( pub struct PduBuilder { #[serde(rename = "type")] pub event_type: EventType, - pub content: serde_json::Value, + pub content: Box, pub unsigned: Option>, pub state_key: Option, pub redacts: Option, @@ -363,8 +394,12 @@ impl From for PduBuilder { fn from(event: AnyInitialStateEvent) -> Self { Self { event_type: EventType::from(event.event_type()), - content: serde_json::value::to_value(event.content()) - .expect("AnyStateEventContent came from JSON and can thus turn back into JSON."), + content: RawValue::from_string( + serde_json::to_string(&event.content()).expect( + "AnyStateEventContent came from JSON and can thus turn back into JSON.", + ), + ) + .expect("string is valid"), unsigned: None, state_key: Some(event.state_key().to_owned()), redacts: None, diff --git a/src/server_server.rs b/src/server_server.rs index 2b8b06c..08ef616 100644 --- a/src/server_server.rs +++ b/src/server_server.rs @@ -39,7 +39,6 @@ use ruma::{ }, directory::{IncomingFilter, IncomingRoomNetwork}, events::{ - pdu::Pdu, receipt::{ReceiptEvent, ReceiptEventContent}, room::{ create::CreateEventContent, @@ -55,7 +54,9 @@ use ruma::{ uint, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName, ServerSigningKeyId, }; +use serde_json::value::RawValue; use std::{ + cell::RefCell, collections::{btree_map, hash_map, BTreeMap, BTreeSet, HashMap, HashSet}, convert::{TryFrom, TryInto}, fmt::Debug, @@ -1095,7 +1096,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( let start_time = Instant::now(); let event_id = pdu.event_id.clone(); if let Err(e) = upgrade_outlier_to_timeline_pdu( - pdu, + &pdu, json, &create_event, origin, @@ -1119,7 +1120,7 @@ pub(crate) async fn handle_incoming_pdu<'a>( } upgrade_outlier_to_timeline_pdu( - incoming_pdu, + &incoming_pdu, val, &create_event, origin, @@ -1139,8 +1140,7 @@ fn handle_outlier_pdu<'a>( value: BTreeMap, db: &'a Database, pub_key_map: &'a RwLock>>, -) -> AsyncRecursiveType<'a, StdResult<(Arc, BTreeMap), String>> -{ +) -> AsyncRecursiveType<'a, StdResult<(PduEvent, BTreeMap), String>> { Box::pin(async move { // TODO: For RoomVersion6 we must check that Raw<..> is canonical do we anywhere?: https://matrix.org/docs/spec/rooms/v6#canonical-json @@ -1153,14 +1153,13 @@ fn handle_outlier_pdu<'a>( // 2. Check signatures, otherwise drop // 3. check content hash, redact if doesn't match - let create_event_content = - serde_json::from_value::>(create_event.content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|e| { - warn!("Invalid create event: {}", e); - "Invalid create event in db.".to_owned() - })?; + let create_event_content = serde_json::from_str::( + create_event.content.get(), + ) + .map_err(|e| { + warn!("Invalid create event: {}", e); + "Invalid create event in db.".to_owned() + })?; let room_version_id = &create_event_content.room_version; let room_version = RoomVersion::new(room_version_id).expect("room version is supported"); @@ -1241,7 +1240,7 @@ fn handle_outlier_pdu<'a>( .expect("all auth events have state keys"), )) { hash_map::Entry::Vacant(v) => { - v.insert(auth_event.clone()); + v.insert(auth_event); } hash_map::Entry::Occupied(_) => { return Err( @@ -1253,11 +1252,7 @@ fn handle_outlier_pdu<'a>( } // The original create event must be in the auth events - if auth_events - .get(&(EventType::RoomCreate, "".to_owned())) - .map(|a| a.as_ref()) - != Some(create_event) - { + if auth_events.get(&(EventType::RoomCreate, "".to_owned())) != Some(create_event) { return Err("Incoming event refers to wrong create event.".to_owned()); } @@ -1268,7 +1263,7 @@ fn handle_outlier_pdu<'a>( db.rooms .get_pdu(&incoming_pdu.auth_events[0]) .map_err(|e| e.to_string())? - .filter(|maybe_create| **maybe_create == *create_event) + .filter(|maybe_create| maybe_create == create_event) } else { None }; @@ -1276,7 +1271,7 @@ fn handle_outlier_pdu<'a>( if !state_res::event_auth::auth_check( &room_version, &incoming_pdu, - previous_create, + previous_create.as_ref(), None::, // TODO: third party invite |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) @@ -1293,13 +1288,13 @@ fn handle_outlier_pdu<'a>( .map_err(|_| "Failed to add pdu as outlier.".to_owned())?; debug!("Added pdu as outlier."); - Ok((Arc::new(incoming_pdu), val)) + Ok((incoming_pdu, val)) }) } #[tracing::instrument(skip(incoming_pdu, val, create_event, origin, db, room_id, pub_key_map))] async fn upgrade_outlier_to_timeline_pdu( - incoming_pdu: Arc, + incoming_pdu: &PduEvent, val: BTreeMap, create_event: &PduEvent, origin: &ServerName, @@ -1320,13 +1315,10 @@ async fn upgrade_outlier_to_timeline_pdu( } let create_event_content = - serde_json::from_value::>(create_event.content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|e| { - warn!("Invalid create event: {}", e); - "Invalid create event in db.".to_owned() - })?; + serde_json::from_str::(create_event.content.get()).map_err(|e| { + warn!("Invalid create event: {}", e); + "Invalid create event in db.".to_owned() + })?; let room_version_id = &create_event_content.room_version; let room_version = RoomVersion::new(room_version_id).expect("room version is supported"); @@ -1434,16 +1426,20 @@ async fn upgrade_outlier_to_timeline_pdu( fork_states.push(state); } + let event_cache = RefCell::new(HashMap::new()); state_at_incoming_event = match state_res::resolve( room_version_id, &fork_states, auth_chain_sets, - |id| { - let res = db.rooms.get_pdu(id); - if let Err(e) = &res { - error!("LOOK AT ME Failed to fetch event: {}", e); + |id| match event_cache.borrow_mut().entry(id.clone()) { + hash_map::Entry::Vacant(v) => { + let res = db.rooms.get_pdu(id); + if let Err(e) = &res { + error!("LOOK AT ME Failed to fetch event: {}", e); + } + v.insert(res.ok().flatten().map(Arc::new)).clone() } - res.ok().flatten() + hash_map::Entry::Occupied(o) => o.get().as_ref().map(Arc::clone), }, ) { Ok(new_state) => Some( @@ -1554,7 +1550,7 @@ async fn upgrade_outlier_to_timeline_pdu( db.rooms .get_pdu(&incoming_pdu.auth_events[0]) .map_err(|e| e.to_string())? - .filter(|maybe_create| **maybe_create == *create_event) + .filter(|maybe_create| maybe_create == create_event) } else { None }; @@ -1562,7 +1558,7 @@ async fn upgrade_outlier_to_timeline_pdu( let check_result = state_res::event_auth::auth_check( &room_version, &incoming_pdu, - previous_create.as_deref(), + previous_create.as_ref(), None::, // TODO: third party invite |k, s| { db.rooms @@ -1646,7 +1642,7 @@ async fn upgrade_outlier_to_timeline_pdu( let soft_fail = !state_res::event_auth::auth_check( &room_version, &incoming_pdu, - previous_create.as_deref(), + previous_create.as_ref(), None::, |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) @@ -1791,16 +1787,20 @@ async fn upgrade_outlier_to_timeline_pdu( .collect::>>() .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?; + let event_cache = RefCell::new(HashMap::new()); let state = match state_res::resolve( room_version_id, fork_states, auth_chain_sets, - |id| { - let res = db.rooms.get_pdu(id); - if let Err(e) = &res { - error!("LOOK AT ME Failed to fetch event: {}", e); + |id| match event_cache.borrow_mut().entry(id.clone()) { + hash_map::Entry::Vacant(v) => { + let res = db.rooms.get_pdu(id); + if let Err(e) = &res { + error!("LOOK AT ME Failed to fetch event: {}", e); + } + v.insert(res.ok().flatten().map(Arc::new)).clone() } - res.ok().flatten() + hash_map::Entry::Occupied(o) => o.get().as_ref().map(Arc::clone), }, ) { Ok(new_state) => new_state, @@ -1873,7 +1873,7 @@ pub(crate) fn fetch_and_handle_outliers<'a>( create_event: &'a PduEvent, room_id: &'a RoomId, pub_key_map: &'a RwLock>>, -) -> AsyncRecursiveType<'a, Vec<(Arc, Option>)>> { +) -> AsyncRecursiveType<'a, Vec<(PduEvent, Option>)>> { Box::pin(async move { let back_off = |id| match db.globals.bad_event_ratelimiter.write().unwrap().entry(id) { hash_map::Entry::Vacant(e) => { @@ -2669,13 +2669,10 @@ pub fn create_join_event_template_route( let create_event_content = create_event .as_ref() .map(|create_event| { - serde_json::from_value::>(create_event.content.clone()) - .expect("Raw::from_value always works.") - .deserialize() - .map_err(|e| { - warn!("Invalid create event: {}", e); - Error::bad_database("Invalid create event in db.") - }) + serde_json::from_str::(create_event.content.get()).map_err(|e| { + warn!("Invalid create event: {}", e); + Error::bad_database("Invalid create event in db.") + }) }) .transpose()?; @@ -2702,16 +2699,19 @@ pub fn create_join_event_template_route( )); } - let content = serde_json::to_value(MemberEventContent { - avatar_url: None, - blurhash: None, - displayname: None, - is_direct: None, - membership: MembershipState::Join, - third_party_invite: None, - reason: None, - }) - .expect("member event is valid value"); + let content = RawValue::from_string( + serde_json::to_string(&MemberEventContent { + avatar_url: None, + blurhash: None, + displayname: None, + is_direct: None, + membership: MembershipState::Join, + third_party_invite: None, + reason: None, + }) + .expect("member event is valid value"), + ) + .expect("string is valid"); let state_key = body.user_id.to_string(); let kind = EventType::RoomMember; @@ -2738,7 +2738,7 @@ pub fn create_join_event_template_route( unsigned.insert("prev_content".to_owned(), prev_pdu.content.clone()); unsigned.insert( "prev_sender".to_owned(), - serde_json::to_value(&prev_pdu.sender).expect("UserId::to_value always works"), + serde_json::from_str(prev_pdu.sender.as_str()).expect("UserId is valid string"), ); } @@ -2751,6 +2751,7 @@ pub fn create_join_event_template_route( .expect("time is valid"), kind, content, + parsed_content: RwLock::new(None), state_key: Some(state_key), prev_events, depth, @@ -2759,17 +2760,26 @@ pub fn create_join_event_template_route( .map(|(_, pdu)| pdu.event_id.clone()) .collect(), redacts: None, - unsigned, + unsigned: if unsigned.is_empty() { + None + } else { + Some( + RawValue::from_string( + serde_json::to_string(&unsigned).expect("to_string always works"), + ) + .expect("string is valid"), + ) + }, hashes: ruma::events::pdu::EventHash { sha256: "aaa".to_owned(), }, - signatures: BTreeMap::new(), + signatures: None, }; let auth_check = state_res::auth_check( &room_version, &pdu, - create_prev_event.as_deref(), + create_prev_event, None::, // TODO: third_party_invite |k, s| auth_events.get(&(k.clone(), s.to_owned())), ) @@ -3279,45 +3289,13 @@ pub(crate) async fn fetch_required_signing_keys( // Gets a list of servers for which we don't have the signing key yet. We go over // the PDUs and either cache the key or add it to the list that needs to be retrieved. -fn get_server_keys_from_cache( - pdu: &Raw, +pub fn get_server_keys_from_cache( + pdu: &CanonicalJsonObject, servers: &mut BTreeMap, BTreeMap>, - room_version: &RoomVersionId, pub_key_map: &mut RwLockWriteGuard<'_, BTreeMap>>, db: &Database, ) -> Result<()> { - let value = serde_json::from_str::(pdu.json().get()).map_err(|e| { - error!("Invalid PDU in server response: {:?}: {:?}", pdu, e); - Error::BadServerResponse("Invalid PDU in server response") - })?; - - let event_id = EventId::try_from(&*format!( - "${}", - ruma::signatures::reference_hash(&value, room_version) - .expect("ruma can calculate reference hashes") - )) - .expect("ruma's reference hashes are valid event ids"); - - if let Some((time, tries)) = db - .globals - .bad_event_ratelimiter - .read() - .unwrap() - .get(&event_id) - { - // Exponential backoff - let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries); - if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) { - min_elapsed_duration = Duration::from_secs(60 * 60 * 24); - } - - if time.elapsed() < min_elapsed_duration { - debug!("Backing off from {}", event_id); - return Err(Error::BadServerResponse("bad event, still backing off")); - } - } - - let signatures = value + let signatures = pdu .get("signatures") .ok_or(Error::BadServerResponse( "No signatures in server response pdu.", @@ -3369,30 +3347,11 @@ fn get_server_keys_from_cache( } pub(crate) async fn fetch_join_signing_keys( - event: &create_join_event::v2::Response, - room_version: &RoomVersionId, + servers: Arc, BTreeMap>>>, pub_key_map: &RwLock>>, db: &Database, ) -> Result<()> { - let mut servers = - BTreeMap::, BTreeMap>::new(); - - { - let mut pkm = pub_key_map - .write() - .map_err(|_| Error::bad_database("RwLock is poisoned."))?; - - // Try to fetch keys, failure is okay - // Servers we couldn't find in the cache will be added to `servers` - for pdu in &event.room_state.state { - let _ = get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm, db); - } - for pdu in &event.room_state.auth_chain { - let _ = get_server_keys_from_cache(pdu, &mut servers, room_version, &mut pkm, db); - } - - drop(pkm); - } + let mut servers = std::mem::take(&mut *servers.write().unwrap()); if servers.is_empty() { // We had all keys locally