WIP

`try_into_http_request` takes care of this already.

.gitignore

@@ -1,5 +1,64 @@
-/target
+# CMake
-**/*.rs.bk
+cmake-build-*/
 
+# IntelliJ
+.idea/
+out/
+*.iml
+modules.xml
+*.ipr
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# Linux backup files
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+# Rust
+/target/
+
+### vscode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows shortcuts
+*.lnk
+
+# Conduit
 Rocket.toml
 conduit.toml
+
+# Etc.
+**/*.rs.bk

.gitlab-ci.yml

@@ -0,0 +1,13 @@
+image: "rust:latest"
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+
+before_script:
+  - apt-get update -yqq
+  - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
+
+test:cargo:
+  script:
+    - rustc --version && cargo --version  # Print version info for debugging
+    - cargo test --workspace --verbose --locked

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "rust-analyzer.procMacro.enable": true
+}

CROSS_COMPILE.md

@@ -1,13 +1,11 @@
 Install docker:
 
+```
 $ sudo apt install docker
 $ sudo usermod -aG docker $USER
-
+$ exec sudo su -l $USER
-Then log out and back in.
-
 $ sudo systemctl start docker
-
 $ cargo install cross
-$ cross build --release --features tls_vendored --target armv7-unknown-linux-musleabihf
+$ cross build --release --target armv7-unknown-linux-musleabihf
-
+```
 The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit

Cargo.toml

@@ -14,25 +14,26 @@ edition = "2018"
 [dependencies]
 # Used to handle requests
 # TODO: This can become optional as soon as proper configs are supported
-rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "93e62c86eddf7cc9a7fc40b044182f83f0d7d92a", features = ["tls"] } # Used to handle requests
+rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "801e04bd5369eb39e126c75f6d11e1e9597304d8", features = ["tls"] } # Used to handle requests
 #rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] }
 
 # Used for matrix spec type definitions and helpers
-#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "appservice-api", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "0a10afe6dacc2b7a50a8002c953d10b7fb4e37bc" }
+ruma = { git = "https://github.com/ruma/ruma", rev = "097f20c837fcbaef1d9c18c30d3ec5a24a6e4ec2", features = ["client-reqwest", "compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "unstable-pre-spec", "unstable-exhaustive-types"] }
-# ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "verified-export" }
+#ruma = { git = "https://github.com/timokoesters/ruma", rev = "220d5b4a76b3b781f7f8297fbe6b14473b04214b", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "unstable-pre-spec", "unstable-exhaustive-types"] }
-ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] }
+#ruma = { path = "../ruma/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "unstable-pre-spec", "unstable-exhaustive-types"] }
 
 # Used when doing state resolution
-# state-res = { git = "https://github.com/timokoesters/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec"] }
+state-res = { git = "https://github.com/ruma/state-res", rev = "bca91bcc17df4d16942ff8b8c3be622b1795e3f0", features = ["unstable-pre-spec"] }
-# TODO: remove the gen-eventid feature
+#state-res = { path = "../state-res", features = ["unstable-pre-spec"] }
-#state-res = { git = "https://github.com/ruma/state-res", branch = "main", features = ["unstable-pre-spec", "gen-eventid"] }
-# state-res = { git = "https://github.com/ruma/state-res", rev = "791c66d73cf064d09db0cdf767d5fef43a343425", features = ["unstable-pre-spec", "gen-eventid"] }
-state-res = { path = "../state-res", features = ["unstable-pre-spec", "gen-eventid"] }
 
 # Used for long polling and federation sender, should be the same as rocket::tokio
 tokio = "1.2.0"
 # Used for storing data permanently
-sled = { version = "0.34.6", default-features = false }
+sled = { version = "0.34.6", features = ["compression", "no_metrics"] }
+#sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] }
+
+# Used for the http request / response body type for Ruma endpoints used with reqwest
+bytes = "1.0.1"
 # Used for emitting log entries
 log = "0.4.14"
 # Used for rocket<->ruma conversions
@@ -50,7 +51,11 @@ rand = "0.8.3"
 # Used to hash passwords
 rust-argon2 = "0.8.3"
 # Used to send requests
-reqwest = { version = "0.11.1", features = ["rustls-tls"] }
+reqwest = { version = "0.11.3", default-features = false, features = ["rustls-tls-native-roots"] }
+# Custom TLS verifier
+rustls = { version = "0.19", features = ["dangerous_configuration"] }
+rustls-native-certs = "0.5.0"
+webpki = "0.21.0"
 # Used for conduit::Error type
 thiserror = "1.0.24"
 # Used to generate thumbnails for images
@@ -76,7 +81,6 @@ pretty_env_logger = "0.4.0"
 [features]
 default = ["conduit_bin"]
 conduit_bin = [] # TODO: add rocket to this when it is optional
-tls_vendored = ["reqwest/native-tls-vendored"]
 
 [[bin]]
 name = "conduit"

DEPLOY.md

@@ -2,7 +2,7 @@
 
 ## Getting help
 
-If you run into any problems while setting up Conduit, write an email to `support@conduit.rs`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
+If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
 
 ## Installing Conduit
 
@@ -18,6 +18,16 @@ $ sudo wget -O /usr/local/bin/matrix-conduit <url>
 $ sudo chmod +x /usr/local/bin/matrix-conduit
 ```
 
+## Adding a Conduit user
+
+While Conduit can run as any user it is usually better to use dedicated users for different services.
+This also allows you to make sure that the file permissions are correctly set up.
+
+In Debian you can use this command to create a Conduit user:
+
+```
+sudo adduser --system conduit --no-create-home
+```
 
 ## Setting up a systemd service
 
@@ -33,8 +43,8 @@ After=network.target
 
 [Service]
 Environment="CONDUIT_CONFIG=/etc/matrix-conduit/conduit.toml"
-User=root
+User=conduit
-Group=root
+Group=nogroup
 Restart=always
 ExecStart=/usr/local/bin/matrix-conduit
 
@@ -43,6 +53,7 @@ WantedBy=multi-user.target
 ```
 
 Finally, run
+
 ```bash
 $ sudo systemctl daemon-reload
 ```
@@ -51,6 +62,7 @@ $ sudo systemctl daemon-reload
 ## Creating the Conduit configuration file
 
 Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment to read it. You need to change at least the server name.**
+
 ```toml
 [global]
 # The server_name is the name of this server. It is used as a suffix for user
@@ -91,6 +103,22 @@ allow_federation = true
 address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy
 ```
 
+## Setting the correct file permissions
+
+As we are using a Conduit specific user we need to allow it to read the config.
+To do that you can run this command on Debian:
+
+```
+sudo chown -R conduit:nogroup /etc/matrix-conduit
+```
+
+If you use the default database path you also need to run this:
+
+```
+sudo mkdir -p /var/lib/matrix-conduit/conduit_db
+sudo chown -R conduit:nogroup /var/lib/matrix-conduit/conduit_db
+```
+
 
 ## Setting up the Reverse Proxy
 
@@ -99,6 +127,7 @@ This depends on whether you use Apache, Nginx or another web server.
 ### Apache
 
 Create `/etc/apache2/sites-enabled/050-conduit.conf` and copy-and-paste this:
+
 ```
 Listen 8448
 
@@ -107,8 +136,8 @@ Listen 8448
 ServerName your.server.name # EDIT THIS
 
 AllowEncodedSlashes NoDecode
-ProxyPass /_matrix/ http://localhost:6167/
+ProxyPass /_matrix/ http://127.0.0.1:6167/_matrix/ nocanon
-ProxyPassReverse /_matrix/ http://localhost:6167/
+ProxyPassReverse /_matrix/ http://127.0.0.1:6167/_matrix/
 
 Include /etc/letsencrypt/options-ssl-apache.conf
 SSLCertificateFile /etc/letsencrypt/live/your.server.name/fullchain.pem # EDIT THIS
@@ -117,6 +146,7 @@ SSLCertificateKeyFile /etc/letsencrypt/live/your.server.name/privkey.pem # EDIT
 ```
 
 **You need to make some edits again.** When you are done, run
+
 ```bash
 $ sudo systemctl reload apache2
 ```
@@ -126,18 +156,30 @@ $ sudo systemctl reload apache2
 
 If you use Nginx and not Apache, add the following server section inside the
 http section of `/etc/nginx/nginx.conf`
+
 ```
 server {
-    listen 443;
+    listen 443 ssl http2;
-    listen 8448;
+    listen [::]:443 ssl http2;
+    listen 8448 ssl http2;
+    listen [::]:8448 ssl http2;
     server_name your.server.name; # EDIT THIS
+    merge_slashes off;
 
     location /_matrix/ {
-        proxy_pass http://localhost:6167/_matrix/;
+        proxy_pass http://127.0.0.1:6167$request_uri;
+        proxy_set_header Host $http_host;
+        proxy_buffering off;
     }
+
+    ssl_certificate /etc/letsencrypt/live/your.server.name/fullchain.pem; # EDIT THIS
+    ssl_certificate_key /etc/letsencrypt/live/your.server.name/privkey.pem; # EDIT THIS
+    ssl_trusted_certificate /etc/letsencrypt/live/your.server.name/chain.pem; # EDIT THIS
+    include /etc/letsencrypt/options-ssl-nginx.conf;
 }
 ```
 **You need to make some edits again.** When you are done, run
+
 ```bash
 $ sudo systemctl reload nginx
 ```
@@ -146,6 +188,7 @@ $ sudo systemctl reload nginx
 ## SSL Certificate
 
 The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this:
+
 ```bash
 $ sudo certbot -d your.server.name
 ```
@@ -154,11 +197,13 @@ $ sudo certbot -d your.server.name
 ## You're done!
 
 Now you can start Conduit with:
+
 ```bash
 $ sudo systemctl start conduit
 ```
 
 Set it to start automatically when your system boots with:
+
 ```bash
 $ sudo systemctl enable conduit
 ```

README.md

@@ -1,25 +1,55 @@
 # Conduit
 ### A Matrix homeserver written in Rust
 
-[![Liberapay](https://img.shields.io/liberapay/receives/timokoesters?logo=liberapay)](https://liberapay.com/timokoesters)
-[![Matrix](https://img.shields.io/matrix/conduit:conduit.rs?server_fqdn=conduit.koesters.xyz)](https://matrix.to/#/#conduit:matrix.org)
-
 #### What is the goal?
 
 A fast Matrix homeserver that's easy to set up and just works. You can install
 it on a mini-computer like the Raspberry Pi to host Matrix for your family,
 friends or company.
 
+
 #### Can I try it out?
 
 Yes! Just open a Matrix client (<https://app.element.io> or Element Android for
 example) and register on the `https://conduit.koesters.xyz` homeserver.
 
+
+#### What is it build on?
+
+- [Ruma](https://www.ruma.io): Useful structures for endpoint requests and
+  responses that can be (de)serialized
+- [Sled](https://github.com/spacejam/sled): A simple (key, value) database with
+  good performance
+- [Rocket](https://rocket.rs): A flexible web framework
+
+
+#### What is the current status?
+
+Conduit can already be used chat with other users on Conduit, chat with users
+from other Matrix servers and even to chat with users on other platforms using
+appservices. When chatting with users on the same Conduit server, everything
+should work assuming you use a compatible client.
+
+**You should not join Matrix rooms without asking the admins first.** We do not
+know whether Conduit is safe for general use yet, so you should assume there is
+some chance that it breaks rooms permanently for all participating users. We
+are not aware of such a bug today, but we would like to do more testing.
+
+There are still a few important features missing:
+
+- Database stability (currently you might have to do manual upgrades or even wipe the db for new versions)
+- End-to-end encrypted chats over federation
+- Typing, presence, read receipts etc. over federation
+- Lots of testing
+
+Check out the [Conduit 1.0 Release Milestone](https://gitlab.com/famedly/conduit/-/milestones/3).
+
+
 #### How can I deploy my own?
 
 ##### Deploy
 
-Download or compile a conduit binary and call it from somewhere like a systemd script. [Read
+Download or compile a Conduit binary, set up the config and call it from somewhere like a systemd script. [Read
 more](DEPLOY.md)
 
 ##### Deploy using Docker
@@ -33,34 +63,23 @@ docker run -d -p 8448:8000 -v db:/srv/conduit/.local/share/conduit matrixconduit
 
 Or build and run it with docker or docker-compose. [Read more](docker/README.md)
 
-#### What is it build on?
-
-- [Ruma](https://www.ruma.io): Useful structures for endpoint requests and
-  responses that can be (de)serialized
-- [Sled](https://github.com/spacejam/sled): A simple (key, value) database with
-  good performance
-- [Rocket](https://rocket.rs): A flexible web framework
-
-#### What are the biggest things still missing?
-
-- Appservices (Bridges and Bots)
-- Most federation features (invites, e2ee)
-- Push notifications on mobile
-- Notification settings
-- Lots of testing
-
-Also check out the [milestones](https://git.koesters.xyz/timo/conduit/milestones).
 
 #### How can I contribute?
 
 1. Look for an issue you would like to work on and make sure it's not assigned
    to other users
 2. Ask someone to assign the issue to you (comment on the issue or chat in
-   #conduit:matrix.org)
+   #conduit:nordgedanken.dev)
-3. Fork the repo and work on the issue. #conduit:matrix.org is happy to help :)
+3. Fork the repo and work on the issue. #conduit:nordgedanken.dev is happy to help :)
-4. Submit a PR
+4. Submit a MR
 
 #### Donate
 
 Liberapay: <https://liberapay.com/timokoesters/>\
 Bitcoin: `bc1qnnykf986tw49ur7wx9rpw2tevpsztvar5x8w4n`
+
+
+#### Logo
+
+Lightning Bolt Logo: https://github.com/mozilla/fxemoji/blob/gh-pages/svgs/nature/u26A1-bolt.svg \
+Logo License: https://github.com/mozilla/fxemoji/blob/gh-pages/LICENSE.md

conduit-example.toml

@@ -35,6 +35,7 @@ max_request_size = 20_000_000 # in bytes
 
 #cache_capacity = 1073741824 # in bytes, 1024 * 1024 * 1024
 #max_concurrent_requests = 4 # How many requests Conduit sends to other servers at the same time
+#log = "info,state_res=warn,rocket=off,_=off,sled=off"
 #workers = 4 # default: cpu core count * 2
 
 address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy

rust-toolchain

@@ -1 +1 @@
-1.47.0
+1.50.0

rustfmt.toml

@@ -1 +1,2 @@
+unstable_features = true
 imports_granularity="Crate"

src/appservice_server.rs

@@ -1,7 +1,10 @@
 use crate::{utils, Error, Result};
-use http::header::{HeaderValue, CONTENT_TYPE};
+use bytes::BytesMut;
 use log::warn;
-use ruma::api::OutgoingRequest;
+use ruma::{
+    api::{IncomingResponse, OutgoingRequest, SendAccessToken},
+    client::HttpClientExt,
+};
 use std::{
     convert::{TryFrom, TryInto},
     fmt::Debug,
@@ -19,86 +22,77 @@ where
     let destination = registration.get("url").unwrap().as_str().unwrap();
     let hs_token = registration.get("hs_token").unwrap().as_str().unwrap();
 
-    let mut http_request = request
+    globals
-        .try_into_http_request(&destination, Some(""))
+        .reqwest_client()
-        .unwrap();
+        .send_customized_matrix_request(
+            &destination,
+            SendAccessToken::IfRequired(""),
+            request,
+            |http_request| {
+                let mut parts = http_request.uri().clone().into_parts();
+                let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned();
+                let symbol = if old_path_and_query.contains('?') {
+                    "&"
+                } else {
+                    "?"
+                };
 
-    let mut parts = http_request.uri().clone().into_parts();
+                parts.path_and_query =
-    let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned();
+                    Some((old_path_and_query + symbol + "access_token=" + hs_token).parse()?);
-    let symbol = if old_path_and_query.contains('?') {
+                *http_request.uri_mut() = parts.try_into()?;
-        "&"
-    } else {
-        "?"
-    };
 
-    parts.path_and_query = Some(
+                Ok(())
-        (old_path_and_query + symbol + "access_token=" + hs_token)
+            },
-            .parse()
+        )
-            .unwrap(),
+        .await
-    );
+        .map_err(|e| {
-    *http_request.uri_mut() = parts.try_into().expect("our manipulation is always valid");
+            warn!("Appservice error: {}", e);
-
+            Error::BadServerResponse("Server returned an error.")
-    http_request.headers_mut().insert(
+        })?;
-        CONTENT_TYPE,
-        HeaderValue::from_str("application/json").unwrap(),
-    );
-
-    let mut reqwest_request = reqwest::Request::try_from(http_request)
-        .expect("all http requests are valid reqwest requests");
 
     *reqwest_request.timeout_mut() = Some(Duration::from_secs(30));
 
     let url = reqwest_request.url().clone();
-    let reqwest_response = globals.reqwest_client().execute(reqwest_request).await;
+    let mut reqwest_response = globals.reqwest_client().execute(reqwest_request).await?;
 
     // Because reqwest::Response -> http::Response is complicated:
-    match reqwest_response {
+    let status = reqwest_response.status();
-        Ok(mut reqwest_response) => {
+    let mut http_response = http::Response::builder().status(status);
-            let status = reqwest_response.status();
+    let headers = http_response.headers_mut().unwrap();
-            let mut http_response = http::Response::builder().status(status);
-            let headers = http_response.headers_mut().unwrap();
 
-            for (k, v) in reqwest_response.headers_mut().drain() {
+    for (k, v) in reqwest_response.headers_mut().drain() {
-                if let Some(key) = k {
+        if let Some(key) = k {
-                    headers.insert(key, v);
+            headers.insert(key, v);
-                }
-            }
-
-            let status = reqwest_response.status();
-
-            let body = reqwest_response
-                .bytes()
-                .await
-                .unwrap_or_else(|e| {
-                    warn!("server error: {}", e);
-                    Vec::new().into()
-                }) // TODO: handle timeout
-                .into_iter()
-                .collect::<Vec<_>>();
-
-            if status != 200 {
-                warn!(
-                    "Appservice returned bad response {} {}\n{}\n{:?}",
-                    destination,
-                    status,
-                    url,
-                    utils::string_from_bytes(&body)
-                );
-            }
-
-            let response = T::IncomingResponse::try_from(
-                http_response
-                    .body(body)
-                    .expect("reqwest body is valid http body"),
-            );
-            response.map_err(|_| {
-                warn!(
-                    "Appservice returned invalid response bytes {}\n{}",
-                    destination, url
-                );
-                Error::BadServerResponse("Server returned bad response.")
-            })
         }
-        Err(e) => Err(e.into()),
     }
+
+    let status = reqwest_response.status();
+
+    let body = reqwest_response.bytes().await.unwrap_or_else(|e| {
+        warn!("server error: {}", e);
+        Vec::new().into()
+    }); // TODO: handle timeout
+
+    if status != 200 {
+        warn!(
+            "Appservice returned bad response {} {}\n{}\n{:?}",
+            destination,
+            status,
+            url,
+            utils::string_from_bytes(&body)
+        );
+    }
+
+    let response = T::IncomingResponse::try_from_http_response(
+        http_response
+            .body(body)
+            .expect("reqwest body is valid http body"),
+    );
+    response.map_err(|_| {
+        warn!(
+            "Appservice returned invalid response bytes {}\n{}",
+            destination, url
+        );
+        Error::BadServerResponse("Server returned bad response.")
+    })
 }

src/client_server/account.rs

@@ -21,7 +21,7 @@ use ruma::{
         },
         EventType,
     },
-    RoomAliasId, RoomId, RoomVersionId, UserId,
+    push, RoomAliasId, RoomId, RoomVersionId, UserId,
 };
 
 use register::RegistrationKind;
@@ -181,7 +181,7 @@ pub async fn register_route(
         EventType::PushRules,
         &ruma::events::push_rules::PushRulesEvent {
             content: ruma::events::push_rules::PushRulesEventContent {
-                global: crate::push_rules::default_pushrules(&user_id),
+                global: push::Ruleset::server_default(&user_id),
             },
         },
         &db.globals,
@@ -617,11 +617,11 @@ pub async fn deactivate_route(
     }
 
     // Leave all joined rooms and reject all invitations
-    for room_id in db
+    for room_id in db.rooms.rooms_joined(&sender_user).chain(
-        .rooms
+        db.rooms
-        .rooms_joined(&sender_user)
+            .rooms_invited(&sender_user)
-        .chain(db.rooms.rooms_invited(&sender_user))
+            .map(|t| t.map(|(r, _)| r)),
-    {
+    ) {
         let room_id = room_id?;
         let event = member::MemberEventContent {
             membership: member::MembershipState::Leave,
@@ -657,3 +657,17 @@ pub async fn deactivate_route(
     }
     .into())
 }
+
+/*/
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/account/3pid", data = "<body>")
+)]
+pub async fn third_party_route(
+    body: Ruma<account::add_3pid::Request<'_>>,
+) -> ConduitResult<account::add_3pid::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    Ok(account::add_3pid::Response::default().into())
+}
+*/

src/client_server/alias.rs

@@ -90,11 +90,23 @@ pub async fn get_alias_helper(
                 let aliases = registration
                     .get("namespaces")
                     .and_then(|ns| ns.get("aliases"))
-                    .and_then(|users| users.get("regex"))
+                    .and_then(|aliases| aliases.as_sequence())
-                    .and_then(|regex| regex.as_str())
+                    .map_or_else(Vec::new, |aliases| {
-                    .and_then(|regex| Regex::new(regex).ok());
+                        aliases
+                            .iter()
+                            .map(|aliases| {
+                                aliases
+                                    .get("regex")
+                                    .and_then(|regex| regex.as_str())
+                                    .and_then(|regex| Regex::new(regex).ok())
+                            })
+                            .filter_map(|o| o)
+                            .collect::<Vec<_>>()
+                    });
 
-                if aliases.map_or(false, |aliases| aliases.is_match(room_alias.as_str()))
+                if aliases
+                    .iter()
+                    .any(|aliases| aliases.is_match(room_alias.as_str()))
                     && db
                         .sending
                         .send_appservice_request(

src/client_server/backup.rs

@@ -267,12 +267,10 @@ pub async fn get_backup_key_session_route(
     let key_data = db
         .key_backups
         .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)?
-        .ok_or_else(|| {
+        .ok_or(Error::BadRequest(
-            Error::BadRequest(
+            ErrorKind::NotFound,
-                ErrorKind::NotFound,
+            "Backup key not found for this user's session.",
-                "Backup key not found for this user's session.",
+        ))?;
-            )
-        })?;
 
     Ok(get_backup_key_session::Response { key_data }.into())
 }

src/client_server/capabilities.rs

@@ -17,7 +17,6 @@ use rocket::get;
 #[tracing::instrument]
 pub async fn get_capabilities_route() -> ConduitResult<get_capabilities::Response> {
     let mut available = BTreeMap::new();
-    available.insert(RoomVersionId::Version5, RoomVersionStability::Stable);
     available.insert(RoomVersionId::Version6, RoomVersionStability::Stable);
 
     let mut capabilities = Capabilities::new();

src/client_server/config.rs

@@ -3,11 +3,16 @@ use crate::{ConduitResult, Database, Error, Ruma};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::config::{get_global_account_data, set_global_account_data},
+        r0::config::{
+            get_global_account_data, get_room_account_data, set_global_account_data,
+            set_room_account_data,
+        },
     },
-    events::{custom::CustomEventContent, BasicEvent},
+    events::{custom::CustomEventContent, AnyBasicEventContent, BasicEvent},
     serde::Raw,
 };
+use serde::Deserialize;
+use serde_json::value::RawValue as RawJsonValue;
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
@@ -43,6 +48,40 @@ pub async fn set_global_account_data_route(
     Ok(set_global_account_data::Response.into())
 }
 
+#[cfg_attr(
+    feature = "conduit_bin",
+    put(
+        "/_matrix/client/r0/user/<_>/rooms/<_>/account_data/<_>",
+        data = "<body>"
+    )
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn set_room_account_data_route(
+    db: State<'_, Database>,
+    body: Ruma<set_room_account_data::Request<'_>>,
+) -> ConduitResult<set_room_account_data::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let data = serde_json::from_str(body.data.get())
+        .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?;
+
+    let event_type = body.event_type.to_string();
+
+    db.account_data.update(
+        Some(&body.room_id),
+        sender_user,
+        event_type.clone().into(),
+        &BasicEvent {
+            content: CustomEventContent { event_type, data },
+        },
+        &db.globals,
+    )?;
+
+    db.flush().await?;
+
+    Ok(set_room_account_data::Response.into())
+}
+
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/user/<_>/account_data/<_>", data = "<body>")
@@ -54,12 +93,51 @@ pub async fn get_global_account_data_route(
 ) -> ConduitResult<get_global_account_data::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    let data = db
+    let event = db
         .account_data
-        .get::<Raw<ruma::events::AnyBasicEvent>>(None, sender_user, body.event_type.clone().into())?
+        .get::<Box<RawJsonValue>>(None, sender_user, body.event_type.clone().into())?
         .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
-
     db.flush().await?;
 
-    Ok(get_global_account_data::Response { account_data: data }.into())
+    let account_data = serde_json::from_str::<ExtractEventContent>(event.get())
+        .expect("event to contain a content field")
+        .content;
+
+    Ok(get_global_account_data::Response { account_data }.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    get(
+        "/_matrix/client/r0/user/<_>/rooms/<_>/account_data/<_>",
+        data = "<body>"
+    )
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_room_account_data_route(
+    db: State<'_, Database>,
+    body: Ruma<get_room_account_data::Request<'_>>,
+) -> ConduitResult<get_room_account_data::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let event = db
+        .account_data
+        .get::<Box<RawJsonValue>>(
+            Some(&body.room_id),
+            sender_user,
+            body.event_type.clone().into(),
+        )?
+        .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
+    db.flush().await?;
+
+    let account_data = serde_json::from_str::<ExtractEventContent>(event.get())
+        .expect("event to contain a content field")
+        .content;
+
+    Ok(get_room_account_data::Response { account_data }.into())
+}
+
+#[derive(Deserialize)]
+struct ExtractEventContent {
+    content: Raw<AnyBasicEventContent>,
 }

src/client_server/context.rs

@@ -24,20 +24,25 @@ pub async fn get_context_route(
         ));
     }
 
+    let base_pdu_id = db
+        .rooms
+        .get_pdu_id(&body.event_id)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "Base event id not found.",
+        ))?;
+
+    let base_token = db.rooms.pdu_count(&base_pdu_id)?;
+
     let base_event = db
         .rooms
-        .get_pdu(&body.event_id)?
+        .get_pdu_from_id(&base_pdu_id)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "Base event not found.",
         ))?
         .to_room_event();
 
-    let base_token = db
-        .rooms
-        .get_pdu_count(&body.event_id)?
-        .expect("event still exists");
-
     let events_before = db
         .rooms
         .pdus_until(&sender_user, &body.room_id, base_token)

src/client_server/directory.rs

@@ -209,17 +209,15 @@ pub async fn get_public_rooms_filtered_helper(
             .map(|room_id| {
                 let room_id = room_id?;
 
-                // TODO: Do not load full state?
-                let state = db.rooms.room_state_full(&room_id)?;
-
                 let chunk = PublicRoomsChunk {
                     aliases: Vec::new(),
-                    canonical_alias: state
+                    canonical_alias: db
-                        .get(&(EventType::RoomCanonicalAlias, "".to_owned()))
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")?
                         .map_or(Ok::<_, Error>(None), |s| {
                             Ok(serde_json::from_value::<
                                 Raw<canonical_alias::CanonicalAliasEventContent>,
-                            >(s.content.clone())
+                            >(s.content)
                             .expect("from_value::<Raw<..>> can never fail")
                             .deserialize()
                             .map_err(|_| {
@@ -227,11 +225,12 @@ pub async fn get_public_rooms_filtered_helper(
                             })?
                             .alias)
                         })?,
-                    name: state.get(&(EventType::RoomName, "".to_owned())).map_or(
+                    name: db
-                        Ok::<_, Error>(None),
+                        .rooms
-                        |s| {
+                        .room_state_get(&room_id, &EventType::RoomName, "")?
+                        .map_or(Ok::<_, Error>(None), |s| {
                             Ok(serde_json::from_value::<Raw<name::NameEventContent>>(
-                                s.content.clone(),
+                                s.content,
                             )
                             .expect("from_value::<Raw<..>> can never fail")
                             .deserialize()
@@ -240,16 +239,15 @@ pub async fn get_public_rooms_filtered_helper(
                             })?
                             .name()
                             .map(|n| n.to_owned()))
-                        },
+                        })?,
-                    )?,
                     num_joined_members: (db.rooms.room_members(&room_id).count() as u32).into(),
-                    room_id,
+                    topic: db
-                    topic: state.get(&(EventType::RoomTopic, "".to_owned())).map_or(
+                        .rooms
-                        Ok::<_, Error>(None),
+                        .room_state_get(&room_id, &EventType::RoomTopic, "")?
-                        |s| {
+                        .map_or(Ok::<_, Error>(None), |s| {
                             Ok(Some(
                                 serde_json::from_value::<Raw<topic::TopicEventContent>>(
-                                    s.content.clone(),
+                                    s.content,
                                 )
                                 .expect("from_value::<Raw<..>> can never fail")
                                 .deserialize()
@@ -258,14 +256,14 @@ pub async fn get_public_rooms_filtered_helper(
                                 })?
                                 .topic,
                             ))
-                        },
+                        })?,
-                    )?,
+                    world_readable: db
-                    world_readable: state
+                        .rooms
-                        .get(&(EventType::RoomHistoryVisibility, "".to_owned()))
+                        .room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")?
                         .map_or(Ok::<_, Error>(false), |s| {
                             Ok(serde_json::from_value::<
                                 Raw<history_visibility::HistoryVisibilityEventContent>,
-                            >(s.content.clone())
+                            >(s.content)
                             .expect("from_value::<Raw<..>> can never fail")
                             .deserialize()
                             .map_err(|_| {
@@ -276,12 +274,13 @@ pub async fn get_public_rooms_filtered_helper(
                             .history_visibility
                                 == history_visibility::HistoryVisibility::WorldReadable)
                         })?,
-                    guest_can_join: state
+                    guest_can_join: db
-                        .get(&(EventType::RoomGuestAccess, "".to_owned()))
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomGuestAccess, "")?
                         .map_or(Ok::<_, Error>(false), |s| {
                             Ok(
                             serde_json::from_value::<Raw<guest_access::GuestAccessEventContent>>(
-                                s.content.clone(),
+                                s.content,
                             )
                             .expect("from_value::<Raw<..>> can never fail")
                             .deserialize()
@@ -292,12 +291,13 @@ pub async fn get_public_rooms_filtered_helper(
                                 == guest_access::GuestAccess::CanJoin,
                         )
                         })?,
-                    avatar_url: state
+                    avatar_url: db
-                        .get(&(EventType::RoomAvatar, "".to_owned()))
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomAvatar, "")?
                         .map(|s| {
                             Ok::<_, Error>(
                                 serde_json::from_value::<Raw<avatar::AvatarEventContent>>(
-                                    s.content.clone(),
+                                    s.content,
                                 )
                                 .expect("from_value::<Raw<..>> can never fail")
                                 .deserialize()
@@ -310,6 +310,7 @@ pub async fn get_public_rooms_filtered_helper(
                         .transpose()?
                         // url is now an Option<String> so we must flatten
                         .flatten(),
+                    room_id,
                 };
                 Ok(chunk)
             })

src/client_server/media.rs

@@ -46,7 +46,7 @@ pub async fn create_content_route(
     db.flush().await?;
 
     Ok(create_content::Response {
-        content_uri: mxc,
+        content_uri: mxc.try_into().expect("Invalid mxc:// URI"),
         blurhash: None,
     }
     .into())

src/client_server/membership.rs

@@ -2,9 +2,11 @@ use super::State;
 use crate::{
     client_server,
     pdu::{PduBuilder, PduEvent},
-    utils, ConduitResult, Database, Error, Result, Ruma,
+    server_server, utils, ConduitResult, Database, Error, Result, Ruma,
 };
-use log::{info, warn};
+use log::{error, warn};
+use member::{MemberEventContent, MembershipState};
+use rocket::futures;
 use ruma::{
     api::{
         client::{
@@ -15,16 +17,21 @@ use ruma::{
                 unban_user, IncomingThirdPartySigned,
             },
         },
-        federation,
+        federation::{self, membership::create_invite},
     },
-    events::{pdu::Pdu, room::member, EventType},
+    events::{
-    serde::{to_canonical_value, CanonicalJsonObject, Raw},
+        pdu::Pdu,
-    EventId, RoomId, RoomVersionId, ServerName, UserId,
+        room::{create::CreateEventContent, member},
+        EventType,
+    },
+    serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw},
+    uint, EventId, RoomId, RoomVersionId, ServerName, UserId,
 };
+use state_res::EventMap;
 use std::{
-    collections::{BTreeMap, HashMap},
+    collections::{BTreeMap, HashSet},
-    convert::TryFrom,
+    convert::{TryFrom, TryInto},
-    sync::Arc,
+    sync::{Arc, RwLock},
 };
 
 #[cfg(feature = "conduit_bin")]
@@ -39,11 +46,29 @@ pub async fn join_room_by_id_route(
     db: State<'_, Database>,
     body: Ruma<join_room_by_id::Request<'_>>,
 ) -> ConduitResult<join_room_by_id::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let mut servers = db
+        .rooms
+        .invite_state(&sender_user, &body.room_id)?
+        .unwrap_or_default()
+        .iter()
+        .filter_map(|event| {
+            serde_json::from_str::<serde_json::Value>(&event.json().to_string()).ok()
+        })
+        .filter_map(|event| event.get("sender").cloned())
+        .filter_map(|sender| sender.as_str().map(|s| s.to_owned()))
+        .filter_map(|sender| UserId::try_from(sender).ok())
+        .map(|user| user.server_name().to_owned())
+        .collect::<HashSet<_>>();
+
+    servers.insert(body.room_id.server_name().to_owned());
+
     join_room_by_id_helper(
         &db,
         body.sender_user.as_ref(),
         &body.room_id,
-        &[body.room_id.server_name().to_owned()],
+        &servers,
         body.third_party_signed.as_ref(),
     )
     .await
@@ -58,12 +83,31 @@ pub async fn join_room_by_id_or_alias_route(
     db: State<'_, Database>,
     body: Ruma<join_room_by_id_or_alias::Request<'_>>,
 ) -> ConduitResult<join_room_by_id_or_alias::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
     let (servers, room_id) = match RoomId::try_from(body.room_id_or_alias.clone()) {
-        Ok(room_id) => (vec![room_id.server_name().to_owned()], room_id),
+        Ok(room_id) => {
+            let mut servers = db
+                .rooms
+                .invite_state(&sender_user, &room_id)?
+                .unwrap_or_default()
+                .iter()
+                .filter_map(|event| {
+                    serde_json::from_str::<serde_json::Value>(&event.json().to_string()).ok()
+                })
+                .filter_map(|event| event.get("sender").cloned())
+                .filter_map(|sender| sender.as_str().map(|s| s.to_owned()))
+                .filter_map(|sender| UserId::try_from(sender).ok())
+                .map(|user| user.server_name().to_owned())
+                .collect::<HashSet<_>>();
+
+            servers.insert(room_id.server_name().to_owned());
+            (servers, room_id)
+        }
         Err(room_alias) => {
             let response = client_server::get_alias_helper(&db, &room_alias).await?;
 
-            (response.0.servers, response.0.room_id)
+            (response.0.servers.into_iter().collect(), response.0.room_id)
         }
     };
 
@@ -95,38 +139,7 @@ pub async fn leave_room_route(
 ) -> ConduitResult<leave_room::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    let mut event = serde_json::from_value::<Raw<member::MemberEventContent>>(
+    db.rooms.leave_room(sender_user, &body.room_id, &db).await?;
-        db.rooms
-            .room_state_get(
-                &body.room_id,
-                &EventType::RoomMember,
-                &sender_user.to_string(),
-            )?
-            .ok_or(Error::BadRequest(
-                ErrorKind::BadState,
-                "Cannot leave a room you are not a member of.",
-            ))?
-            .1
-            .content,
-    )
-    .expect("from_value::<Raw<..>> can never fail")
-    .deserialize()
-    .map_err(|_| Error::bad_database("Invalid member event in database."))?;
-
-    event.membership = member::MembershipState::Leave;
-
-    db.rooms.build_and_append_pdu(
-        PduBuilder {
-            event_type: EventType::RoomMember,
-            content: serde_json::to_value(event).expect("event is valid, we just created it"),
-            unsigned: None,
-            state_key: Some(sender_user.to_string()),
-            redacts: None,
-        },
-        &sender_user,
-        &body.room_id,
-        &db,
-    )?;
 
     db.flush().await?;
 
@@ -145,28 +158,8 @@ pub async fn invite_user_route(
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     if let invite_user::IncomingInvitationRecipient::UserId { user_id } = &body.recipient {
-        db.rooms.build_and_append_pdu(
+        invite_helper(sender_user, user_id, &body.room_id, &db, false).await?;
-            PduBuilder {
-                event_type: EventType::RoomMember,
-                content: serde_json::to_value(member::MemberEventContent {
-                    membership: member::MembershipState::Invite,
-                    displayname: db.users.displayname(&user_id)?,
-                    avatar_url: db.users.avatar_url(&user_id)?,
-                    is_direct: None,
-                    third_party_invite: None,
-                })
-                .expect("event is valid, we just created it"),
-                unsigned: None,
-                state_key: Some(user_id.to_string()),
-                redacts: None,
-            },
-            &sender_user,
-            &body.room_id,
-            &db,
-        )?;
-
         db.flush().await?;
-
         Ok(invite_user::Response.into())
     } else {
         Err(Error::BadRequest(ErrorKind::NotFound, "User not found."))
@@ -195,7 +188,6 @@ pub async fn kick_user_route(
                 ErrorKind::BadState,
                 "Cannot kick member that's not in the room.",
             ))?
-            .1
             .content,
     )
     .expect("Raw::from_value always works")
@@ -251,7 +243,7 @@ pub async fn ban_user_route(
                 is_direct: None,
                 third_party_invite: None,
             }),
-            |(_, event)| {
+            |event| {
                 let mut event =
                     serde_json::from_value::<Raw<member::MemberEventContent>>(event.content)
                         .expect("Raw::from_value always works")
@@ -302,7 +294,6 @@ pub async fn unban_user_route(
                 ErrorKind::BadState,
                 "Cannot unban a user who is not banned.",
             ))?
-            .1
             .content,
     )
     .expect("from_value::<Raw<..>> can never fail")
@@ -442,7 +433,7 @@ async fn join_room_by_id_helper(
     db: &Database,
     sender_user: Option<&UserId>,
     room_id: &RoomId,
-    servers: &[Box<ServerName>],
+    servers: &HashSet<Box<ServerName>>,
     _third_party_signed: Option<&IncomingThirdPartySigned>,
 ) -> ConduitResult<join_room_by_id::Response> {
     let sender_user = sender_user.expect("user is authenticated");
@@ -462,7 +453,7 @@ async fn join_room_by_id_helper(
                     federation::membership::create_join_event_template::v1::Request {
                         room_id,
                         user_id: sender_user,
-                        ver: &[RoomVersionId::Version5, RoomVersionId::Version6],
+                        ver: &[RoomVersionId::Version6],
                     },
                 )
                 .await;
@@ -476,21 +467,29 @@ async fn join_room_by_id_helper(
 
         let (make_join_response, remote_server) = make_join_response_and_server?;
 
+        let room_version = match make_join_response.room_version {
+            Some(room_version) if room_version == RoomVersionId::Version6 => room_version,
+            _ => return Err(Error::BadServerResponse("Room version is not supported")),
+        };
+
         let mut join_event_stub =
             serde_json::from_str::<CanonicalJsonObject>(make_join_response.event.json().get())
                 .map_err(|_| {
                     Error::BadServerResponse("Invalid make_join event json received from server.")
                 })?;
 
+        // TODO: Is origin needed?
         join_event_stub.insert(
             "origin".to_owned(),
-            to_canonical_value(db.globals.server_name())
+            CanonicalJsonValue::String(db.globals.server_name().as_str().to_owned()),
-                .map_err(|_| Error::bad_database("Invalid server name found"))?,
         );
         join_event_stub.insert(
             "origin_server_ts".to_owned(),
-            to_canonical_value(utils::millis_since_unix_epoch())
+            CanonicalJsonValue::Integer(
-                .expect("Timestamp is valid js_int value"),
+                utils::millis_since_unix_epoch()
+                    .try_into()
+                    .expect("Timestamp is valid js_int value"),
+            ),
         );
         join_event_stub.insert(
             "content".to_owned(),
@@ -512,14 +511,14 @@ async fn join_room_by_id_helper(
             db.globals.server_name().as_str(),
             db.globals.keypair(),
             &mut join_event_stub,
-            &RoomVersionId::Version6,
+            &room_version,
         )
         .expect("event is valid, we just created it");
 
         // Generate event id
         let event_id = EventId::try_from(&*format!(
             "${}",
-            ruma::signatures::reference_hash(&join_event_stub, &RoomVersionId::Version6)
+            ruma::signatures::reference_hash(&join_event_stub, &room_version)
                 .expect("ruma can calculate reference hashes")
         ))
         .expect("ruma's reference hashes are valid event ids");
@@ -527,7 +526,7 @@ async fn join_room_by_id_helper(
         // Add event_id back
         join_event_stub.insert(
             "event_id".to_owned(),
-            to_canonical_value(&event_id).expect("EventId is a valid CanonicalJsonValue"),
+            CanonicalJsonValue::String(event_id.as_str().to_owned()),
         );
 
         // It has enough fields to be called a proper event now
@@ -546,25 +545,6 @@ async fn join_room_by_id_helper(
             )
             .await?;
 
-        let add_event_id = |pdu: &Raw<Pdu>| -> Result<(EventId, CanonicalJsonObject)> {
-            let mut value = serde_json::from_str(pdu.json().get())
-                .expect("converting raw jsons to values always works");
-            let event_id = EventId::try_from(&*format!(
-                "${}",
-                ruma::signatures::reference_hash(&value, &RoomVersionId::Version6)
-                    .expect("ruma can calculate reference hashes")
-            ))
-            .expect("ruma's reference hashes are valid event ids");
-
-            value.insert(
-                "event_id".to_owned(),
-                to_canonical_value(&event_id)
-                    .expect("a valid EventId can be converted to CanonicalJsonValue"),
-            );
-
-            Ok((event_id, value))
-        };
-
         let count = db.globals.next_count()?;
 
         let mut pdu_id = room_id.as_bytes().to_vec();
@@ -572,55 +552,63 @@ async fn join_room_by_id_helper(
         pdu_id.extend_from_slice(&count.to_be_bytes());
 
         let pdu = PduEvent::from_id_val(&event_id, join_event.clone())
-            .map_err(|_| Error::BadServerResponse("Invalid PDU in send_join response."))?;
+            .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?;
 
-        let mut state = HashMap::new();
+        let mut state = BTreeMap::new();
+        let pub_key_map = RwLock::new(BTreeMap::new());
 
-        for pdu in send_join_response
+        for result in futures::future::join_all(
-            .room_state
+            send_join_response
-            .state
+                .room_state
-            .iter()
+                .state
-            .map(add_event_id)
+                .iter()
-            .map(|r| {
+                .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db)),
-                let (event_id, value) = r?;
+        )
-                PduEvent::from_id_val(&event_id, value.clone())
+        .await
-                    .map(|ev| (event_id, Arc::new(ev)))
-                    .map_err(|e| {
-                        warn!("{:?}: {}", value, e);
-                        Error::BadServerResponse("Invalid PDU in send_join response.")
-                    })
-            })
         {
-            let (id, pdu) = pdu?;
+            let (event_id, value) = match result {
-            info!("adding {} to outliers: {:#?}", id, pdu);
+                Ok(t) => t,
-            db.rooms.add_pdu_outlier(&pdu)?;
+                Err(_) => continue,
+            };
+
+            let pdu = PduEvent::from_id_val(&event_id, value.clone()).map_err(|e| {
+                warn!("{:?}: {}", value, e);
+                Error::BadServerResponse("Invalid PDU in send_join response.")
+            })?;
+
+            db.rooms.add_pdu_outlier(&event_id, &value)?;
             if let Some(state_key) = &pdu.state_key {
                 if pdu.kind == EventType::RoomMember {
-                    let target_user_id = UserId::try_from(state_key.clone()).map_err(|_| {
+                    let target_user_id = UserId::try_from(state_key.clone()).map_err(|e| {
+                        warn!(
+                            "Invalid user id in send_join response: {}: {}",
+                            state_key, e
+                        );
                         Error::BadServerResponse("Invalid user id in send_join response.")
                     })?;
 
+                    let invite_state = Vec::new(); // TODO add a few important events
+
                     // Update our membership info, we do this here incase a user is invited
                     // and immediately leaves we need the DB to record the invite event for auth
                     db.rooms.update_membership(
                         &pdu.room_id,
                         &target_user_id,
-                        serde_json::from_value::<member::MemberEventContent>(pdu.content.clone())
+                        serde_json::from_value::<member::MembershipState>(
-                            .map_err(|_| {
+                            pdu.content
-                            Error::BadRequest(
+                                .get("membership")
-                                ErrorKind::InvalidParam,
+                                .ok_or(Error::BadServerResponse("Invalid member event content"))?
-                                "Invalid member event content.",
+                                .clone(),
-                            )
+                        )
+                        .map_err(|_| {
+                            Error::BadServerResponse("Invalid membership state content.")
                         })?,
                         &pdu.sender,
-                        &db.account_data,
+                        Some(invite_state),
-                        &db.globals,
+                        db,
                     )?;
                 }
-                let mut long_id = room_id.as_bytes().to_vec();
+                state.insert((pdu.kind.clone(), state_key.clone()), pdu.event_id.clone());
-                long_id.push(0xff);
-                long_id.extend_from_slice(id.as_bytes());
-                state.insert((pdu.kind.clone(), state_key.clone()), long_id);
             }
         }
 
@@ -629,31 +617,36 @@ async fn join_room_by_id_helper(
                 pdu.kind.clone(),
                 pdu.state_key.clone().expect("join event has state key"),
             ),
-            pdu_id.clone(),
+            pdu.event_id.clone(),
         );
 
-        db.rooms.force_state(room_id, state, &db.globals)?;
+        if state.get(&(EventType::RoomCreate, "".to_owned())).is_none() {
-
+            return Err(Error::BadServerResponse("State contained no create event."));
-        for pdu in send_join_response
-            .room_state
-            .auth_chain
-            .iter()
-            .map(add_event_id)
-            .map(|r| {
-                let (event_id, value) = r?;
-                PduEvent::from_id_val(&event_id, value.clone())
-                    .map(|ev| (event_id, Arc::new(ev)))
-                    .map_err(|e| {
-                        warn!("{:?}: {}", value, e);
-                        Error::BadServerResponse("Invalid PDU in send_join response.")
-                    })
-            })
-        {
-            let (id, pdu) = pdu?;
-            info!("adding {} to outliers: {:#?}", id, pdu);
-            db.rooms.add_pdu_outlier(&pdu)?;
         }
 
+        db.rooms.force_state(room_id, state, &db)?;
+
+        for result in futures::future::join_all(
+            send_join_response
+                .room_state
+                .auth_chain
+                .iter()
+                .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db)),
+        )
+        .await
+        {
+            let (event_id, value) = match result {
+                Ok(t) => t,
+                Err(_) => continue,
+            };
+
+            db.rooms.add_pdu_outlier(&event_id, &value)?;
+        }
+
+        // We append to state before appending the pdu, so we don't have a moment in time with the
+        // pdu without it's state. This is okay because append_pdu can't fail.
+        let statehashid = db.rooms.append_to_state(&pdu, &db.globals)?;
+
         db.rooms.append_pdu(
             &pdu,
             utils::to_canonical_object(&pdu).expect("Pdu is valid canonical object"),
@@ -662,6 +655,10 @@ async fn join_room_by_id_helper(
             &[pdu.event_id.clone()],
             db,
         )?;
+
+        // We set the room state after inserting the pdu, so that we never have a moment in time
+        // where events in the current room state do not exist
+        db.rooms.set_room_state(&room_id, statehashid)?;
     } else {
         let event = member::MemberEventContent {
             membership: member::MembershipState::Join,
@@ -685,5 +682,283 @@ async fn join_room_by_id_helper(
         )?;
     }
 
+    db.flush().await?;
+
     Ok(join_room_by_id::Response::new(room_id.clone()).into())
 }
+
+async fn validate_and_add_event_id(
+    pdu: &Raw<Pdu>,
+    room_version: &RoomVersionId,
+    pub_key_map: &RwLock<BTreeMap<String, BTreeMap<String, String>>>,
+    db: &Database,
+) -> Result<(EventId, CanonicalJsonObject)> {
+    let mut value = serde_json::from_str::<CanonicalJsonObject>(pdu.json().get()).map_err(|e| {
+        error!("{:?}: {:?}", pdu, e);
+        Error::BadServerResponse("Invalid PDU in server response")
+    })?;
+
+    server_server::fetch_required_signing_keys(&value, pub_key_map, db).await?;
+    if let Err(e) = ruma::signatures::verify_event(
+        &*pub_key_map
+            .read()
+            .map_err(|_| Error::bad_database("RwLock is poisoned."))?,
+        &value,
+        room_version,
+    ) {
+        warn!("Event failed verification: {}", e);
+        return Err(Error::BadServerResponse("Event failed verification."));
+    }
+
+    let event_id = EventId::try_from(&*format!(
+        "${}",
+        ruma::signatures::reference_hash(&value, &room_version)
+            .expect("ruma can calculate reference hashes")
+    ))
+    .expect("ruma's reference hashes are valid event ids");
+
+    value.insert(
+        "event_id".to_owned(),
+        CanonicalJsonValue::String(event_id.as_str().to_owned()),
+    );
+
+    Ok((event_id, value))
+}
+
+pub async fn invite_helper(
+    sender_user: &UserId,
+    user_id: &UserId,
+    room_id: &RoomId,
+    db: &Database,
+    is_direct: bool,
+) -> Result<()> {
+    if user_id.server_name() != db.globals.server_name() {
+        let prev_events = db
+            .rooms
+            .get_pdu_leaves(room_id)?
+            .into_iter()
+            .take(20)
+            .collect::<Vec<_>>();
+
+        let create_event = db
+            .rooms
+            .room_state_get(room_id, &EventType::RoomCreate, "")?;
+
+        let create_event_content = create_event
+            .as_ref()
+            .map(|create_event| {
+                Ok::<_, Error>(
+                    serde_json::from_value::<Raw<CreateEventContent>>(create_event.content.clone())
+                        .expect("Raw::from_value always works.")
+                        .deserialize()
+                        .map_err(|_| Error::bad_database("Invalid PowerLevels event in db."))?,
+                )
+            })
+            .transpose()?;
+
+        let create_prev_event = if prev_events.len() == 1
+            && Some(&prev_events[0]) == create_event.as_ref().map(|c| &c.event_id)
+        {
+            create_event.map(Arc::new)
+        } else {
+            None
+        };
+
+        // If there was no create event yet, assume we are creating a version 6 room right now
+        let room_version = create_event_content.map_or(RoomVersionId::Version6, |create_event| {
+            create_event.room_version
+        });
+
+        let content = serde_json::to_value(MemberEventContent {
+            avatar_url: None,
+            displayname: None,
+            is_direct: Some(is_direct),
+            membership: MembershipState::Invite,
+            third_party_invite: None,
+        })
+        .expect("member event is valid value");
+
+        let state_key = user_id.to_string();
+        let kind = EventType::RoomMember;
+
+        let auth_events =
+            db.rooms
+                .get_auth_events(room_id, &kind, &sender_user, Some(&state_key), &content)?;
+
+        // Our depth is the maximum depth of prev_events + 1
+        let depth = prev_events
+            .iter()
+            .filter_map(|event_id| Some(db.rooms.get_pdu(event_id).ok()??.depth))
+            .max()
+            .unwrap_or_else(|| uint!(0))
+            + uint!(1);
+
+        let mut unsigned = BTreeMap::new();
+
+        if let Some(prev_pdu) = db.rooms.room_state_get(room_id, &kind, &state_key)? {
+            unsigned.insert("prev_content".to_owned(), prev_pdu.content);
+            unsigned.insert(
+                "prev_sender".to_owned(),
+                serde_json::to_value(prev_pdu.sender).expect("UserId::to_value always works"),
+            );
+        }
+
+        let pdu = PduEvent {
+            event_id: ruma::event_id!("$thiswillbefilledinlater"),
+            room_id: room_id.clone(),
+            sender: sender_user.clone(),
+            origin_server_ts: utils::millis_since_unix_epoch()
+                .try_into()
+                .expect("time is valid"),
+            kind,
+            content,
+            state_key: Some(state_key),
+            prev_events,
+            depth,
+            auth_events: auth_events
+                .iter()
+                .map(|(_, pdu)| pdu.event_id.clone())
+                .collect(),
+            redacts: None,
+            unsigned,
+            hashes: ruma::events::pdu::EventHash {
+                sha256: "aaa".to_owned(),
+            },
+            signatures: BTreeMap::new(),
+        };
+
+        let auth_check = state_res::auth_check(
+            &room_version,
+            &Arc::new(pdu.clone()),
+            create_prev_event,
+            &auth_events,
+            None, // TODO: third_party_invite
+        )
+        .map_err(|e| {
+            error!("{:?}", e);
+            Error::bad_database("Auth check failed.")
+        })?;
+
+        if !auth_check {
+            return Err(Error::BadRequest(
+                ErrorKind::InvalidParam,
+                "Event is not authorized.",
+            ));
+        }
+
+        // Hash and sign
+        let mut pdu_json =
+            utils::to_canonical_object(&pdu).expect("event is valid, we just created it");
+
+        pdu_json.remove("event_id");
+
+        // Add origin because synapse likes that (and it's required in the spec)
+        pdu_json.insert(
+            "origin".to_owned(),
+            to_canonical_value(db.globals.server_name())
+                .expect("server name is a valid CanonicalJsonValue"),
+        );
+
+        ruma::signatures::hash_and_sign_event(
+            db.globals.server_name().as_str(),
+            db.globals.keypair(),
+            &mut pdu_json,
+            &room_version,
+        )
+        .expect("event is valid, we just created it");
+
+        let invite_room_state = db.rooms.calculate_invite_state(&pdu)?;
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                user_id.server_name(),
+                create_invite::v2::Request {
+                    room_id: room_id.clone(),
+                    event_id: ruma::event_id!("$receivingservershouldsetthis"),
+                    room_version: RoomVersionId::Version6,
+                    event: PduEvent::convert_to_outgoing_federation_event(pdu_json),
+                    invite_room_state,
+                },
+            )
+            .await?;
+
+        let pub_key_map = RwLock::new(BTreeMap::new());
+        let mut auth_cache = EventMap::new();
+
+        // We do not add the event_id field to the pdu here because of signature and hashes checks
+        let (event_id, value) = match crate::pdu::gen_event_id_canonical_json(&response.event) {
+            Ok(t) => t,
+            Err(_) => {
+                // Event could not be converted to canonical json
+                return Err(Error::BadRequest(
+                    ErrorKind::InvalidParam,
+                    "Could not convert event to canonical json.",
+                ));
+            }
+        };
+
+        let origin = serde_json::from_value::<Box<ServerName>>(
+            serde_json::to_value(value.get("origin").ok_or(Error::BadRequest(
+                ErrorKind::InvalidParam,
+                "Event needs an origin field.",
+            ))?)
+            .expect("CanonicalJson is valid json value"),
+        )
+        .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Origin field is invalid."))?;
+
+        let pdu_id = server_server::handle_incoming_pdu(
+            &origin,
+            &event_id,
+            value,
+            true,
+            &db,
+            &pub_key_map,
+            &mut auth_cache,
+        )
+        .await
+        .map_err(|_| {
+            Error::BadRequest(
+                ErrorKind::InvalidParam,
+                "Error while handling incoming PDU.",
+            )
+        })?
+        .ok_or(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Could not accept incoming PDU as timeline event.",
+        ))?;
+
+        for server in db
+            .rooms
+            .room_servers(room_id)
+            .filter_map(|r| r.ok())
+            .filter(|server| &**server != db.globals.server_name())
+        {
+            db.sending.send_pdu(&server, &pdu_id)?;
+        }
+
+        return Ok(());
+    }
+
+    db.rooms.build_and_append_pdu(
+        PduBuilder {
+            event_type: EventType::RoomMember,
+            content: serde_json::to_value(member::MemberEventContent {
+                membership: member::MembershipState::Invite,
+                displayname: db.users.displayname(&user_id)?,
+                avatar_url: db.users.avatar_url(&user_id)?,
+                is_direct: None,
+                third_party_invite: None,
+            })
+            .expect("event is valid, we just created it"),
+            unsigned: None,
+            state_key: Some(user_id.to_string()),
+            redacts: None,
+        },
+        &sender_user,
+        room_id,
+        &db,
+    )?;
+
+    Ok(())
+}

src/client_server/message.rs

@@ -5,7 +5,7 @@ use ruma::{
         error::ErrorKind,
         r0::message::{get_message_events, send_message_event},
     },
-    events::EventContent,
+    events::EventType,
     EventId,
 };
 use std::{
@@ -55,7 +55,7 @@ pub async fn send_message_event_route(
 
     let event_id = db.rooms.build_and_append_pdu(
         PduBuilder {
-            event_type: body.content.event_type().into(),
+            event_type: EventType::from(&body.event_type),
             content: serde_json::from_str(
                 body.json_body
                     .as_ref()

src/client_server/profile.rs

@@ -30,42 +30,48 @@ pub async fn set_displayname_route(
         .set_displayname(&sender_user, body.displayname.clone())?;
 
     // Send a new membership event and presence update into all joined rooms
-    for room_id in db.rooms.rooms_joined(&sender_user) {
+    for (pdu_builder, room_id) in db
-        let room_id = room_id?;
+        .rooms
-        db.rooms.build_and_append_pdu(
+        .rooms_joined(&sender_user)
-            PduBuilder {
+        .filter_map(|r| r.ok())
-                event_type: EventType::RoomMember,
+        .map(|room_id| {
-                content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
+            Ok::<_, Error>((
-                    displayname: body.displayname.clone(),
+                PduBuilder {
-                    ..serde_json::from_value::<Raw<_>>(
+                    event_type: EventType::RoomMember,
-                        db.rooms
+                    content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
-                            .room_state_get(
+                        displayname: body.displayname.clone(),
-                                &room_id,
+                        ..serde_json::from_value::<Raw<_>>(
-                                &EventType::RoomMember,
+                            db.rooms
-                                &sender_user.to_string(),
+                                .room_state_get(
-                            )?
+                                    &room_id,
-                            .ok_or_else(|| {
+                                    &EventType::RoomMember,
-                                Error::bad_database(
+                                    &sender_user.to_string(),
-                                    "Tried to send displayname update for user not in the room.",
+                                )?
-                                )
+                                .ok_or_else(|| {
-                            })?
+                                    Error::bad_database(
-                            .1
+                                        "Tried to send displayname update for user not in the \
-                            .content
+                                         room.",
-                            .clone(),
+                                    )
-                    )
+                                })?
-                    .expect("from_value::<Raw<..>> can never fail")
+                                .content,
-                    .deserialize()
+                        )
-                    .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
+                        .expect("from_value::<Raw<..>> can never fail")
-                })
+                        .deserialize()
-                .expect("event is valid, we just created it"),
+                        .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
-                unsigned: None,
+                    })
-                state_key: Some(sender_user.to_string()),
+                    .expect("event is valid, we just created it"),
-                redacts: None,
+                    unsigned: None,
-            },
+                    state_key: Some(sender_user.to_string()),
-            &sender_user,
+                    redacts: None,
-            &room_id,
+                },
-            &db,
+                room_id,
-        )?;
+            ))
+        })
+        .filter_map(|r| r.ok())
+    {
+        let _ = db
+            .rooms
+            .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db);
 
         // Presence update
         db.rooms.edus.update_presence(
@@ -125,42 +131,48 @@ pub async fn set_avatar_url_route(
         .set_avatar_url(&sender_user, body.avatar_url.clone())?;
 
     // Send a new membership event and presence update into all joined rooms
-    for room_id in db.rooms.rooms_joined(&sender_user) {
+    for (pdu_builder, room_id) in db
-        let room_id = room_id?;
+        .rooms
-        db.rooms.build_and_append_pdu(
+        .rooms_joined(&sender_user)
-            PduBuilder {
+        .filter_map(|r| r.ok())
-                event_type: EventType::RoomMember,
+        .map(|room_id| {
-                content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
+            Ok::<_, Error>((
-                    avatar_url: body.avatar_url.clone(),
+                PduBuilder {
-                    ..serde_json::from_value::<Raw<_>>(
+                    event_type: EventType::RoomMember,
-                        db.rooms
+                    content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
-                            .room_state_get(
+                        avatar_url: body.avatar_url.clone(),
-                                &room_id,
+                        ..serde_json::from_value::<Raw<_>>(
-                                &EventType::RoomMember,
+                            db.rooms
-                                &sender_user.to_string(),
+                                .room_state_get(
-                            )?
+                                    &room_id,
-                            .ok_or_else(|| {
+                                    &EventType::RoomMember,
-                                Error::bad_database(
+                                    &sender_user.to_string(),
-                                    "Tried to send avatar url update for user not in the room.",
+                                )?
-                                )
+                                .ok_or_else(|| {
-                            })?
+                                    Error::bad_database(
-                            .1
+                                        "Tried to send displayname update for user not in the \
-                            .content
+                                         room.",
-                            .clone(),
+                                    )
-                    )
+                                })?
-                    .expect("from_value::<Raw<..>> can never fail")
+                                .content,
-                    .deserialize()
+                        )
-                    .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
+                        .expect("from_value::<Raw<..>> can never fail")
-                })
+                        .deserialize()
-                .expect("event is valid, we just created it"),
+                        .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
-                unsigned: None,
+                    })
-                state_key: Some(sender_user.to_string()),
+                    .expect("event is valid, we just created it"),
-                redacts: None,
+                    unsigned: None,
-            },
+                    state_key: Some(sender_user.to_string()),
-            &sender_user,
+                    redacts: None,
-            &room_id,
+                },
-            &db,
+                room_id,
-        )?;
+            ))
+        })
+        .filter_map(|r| r.ok())
+    {
+        let _ = db
+            .rooms
+            .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db);
 
         // Presence update
         db.rooms.edus.update_presence(

src/client_server/push.rs

@@ -5,14 +5,12 @@ use ruma::{
         error::ErrorKind,
         r0::push::{
             delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled,
-            get_pushrules_all, set_pushrule, set_pushrule_actions, set_pushrule_enabled, RuleKind,
+            get_pushrules_all, set_pusher, set_pushrule, set_pushrule_actions,
+            set_pushrule_enabled, RuleKind,
         },
     },
-    events::EventType,
+    events::{push_rules, EventType},
-    push::{
+    push::{ConditionalPushRuleInit, PatternedPushRuleInit, SimplePushRuleInit},
-        ConditionalPushRuleInit, ContentPushRule, OverridePushRule, PatternedPushRuleInit,
-        RoomPushRule, SenderPushRule, SimplePushRuleInit, UnderridePushRule,
-    },
 };
 
 #[cfg(feature = "conduit_bin")]
@@ -31,7 +29,7 @@ pub async fn get_pushrules_all_route(
 
     let event = db
         .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "PushRules event not found.",
@@ -56,7 +54,7 @@ pub async fn get_pushrule_route(
 
     let event = db
         .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "PushRules event not found.",
@@ -66,29 +64,24 @@ pub async fn get_pushrule_route(
     let rule = match body.kind {
         RuleKind::Override => global
             .override_
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.clone().into()),
-            .map(|rule| rule.0.clone().into()),
         RuleKind::Underride => global
             .underride
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.clone().into()),
-            .map(|rule| rule.0.clone().into()),
         RuleKind::Sender => global
             .sender
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.clone().into()),
-            .map(|rule| rule.0.clone().into()),
         RuleKind::Room => global
             .room
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.clone().into()),
-            .map(|rule| rule.0.clone().into()),
         RuleKind::Content => global
             .content
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.clone().into()),
-            .map(|rule| rule.0.clone().into()),
         RuleKind::_Custom(_) => None,
     };
 
@@ -104,14 +97,15 @@ pub async fn get_pushrule_route(
 
 #[cfg_attr(
     feature = "conduit_bin",
-    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
+    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<req>")
 )]
-#[tracing::instrument(skip(db, body))]
+#[tracing::instrument(skip(db, req))]
 pub async fn set_pushrule_route(
     db: State<'_, Database>,
-    body: Ruma<set_pushrule::Request<'_>>,
+    req: Ruma<set_pushrule::Request<'_>>,
 ) -> ConduitResult<set_pushrule::Response> {
-    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+    let sender_user = req.sender_user.as_ref().expect("user is authenticated");
+    let body = req.body;
 
     if body.scope != "global" {
         return Err(Error::BadRequest(
@@ -122,7 +116,7 @@ pub async fn set_pushrule_route(
 
     let mut event = db
         .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "PushRules event not found.",
@@ -131,107 +125,62 @@ pub async fn set_pushrule_route(
     let global = &mut event.content.global;
     match body.kind {
         RuleKind::Override => {
-            if let Some(rule) = global
+            global.override_.replace(
-                .override_
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.override_.remove(&rule);
-            }
-
-            global.override_.insert(OverridePushRule(
                 ConditionalPushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                     default: false,
                     enabled: true,
-                    rule_id: body.rule_id.clone(),
+                    rule_id: body.rule_id,
-                    conditions: body.conditions.clone(),
+                    conditions: body.conditions,
                 }
                 .into(),
-            ));
+            );
         }
         RuleKind::Underride => {
-            if let Some(rule) = global
+            global.underride.replace(
-                .underride
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.underride.remove(&rule);
-            }
-
-            global.underride.insert(UnderridePushRule(
                 ConditionalPushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                     default: false,
                     enabled: true,
-                    rule_id: body.rule_id.clone(),
+                    rule_id: body.rule_id,
-                    conditions: body.conditions.clone(),
+                    conditions: body.conditions,
                 }
                 .into(),
-            ));
+            );
         }
         RuleKind::Sender => {
-            if let Some(rule) = global
+            global.sender.replace(
-                .sender
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.sender.remove(&rule);
-            }
-
-            global.sender.insert(SenderPushRule(
                 SimplePushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                     default: false,
                     enabled: true,
-                    rule_id: body.rule_id.clone(),
+                    rule_id: body.rule_id,
                 }
                 .into(),
-            ));
+            );
         }
         RuleKind::Room => {
-            if let Some(rule) = global
+            global.room.replace(
-                .room
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.room.remove(&rule);
-            }
-
-            global.room.insert(RoomPushRule(
                 SimplePushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                     default: false,
                     enabled: true,
-                    rule_id: body.rule_id.clone(),
+                    rule_id: body.rule_id,
                 }
                 .into(),
-            ));
+            );
         }
         RuleKind::Content => {
-            if let Some(rule) = global
+            global.content.replace(
-                .content
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.content.remove(&rule);
-            }
-
-            global.content.insert(ContentPushRule(
                 PatternedPushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                     default: false,
                     enabled: true,
-                    rule_id: body.rule_id.clone(),
+                    rule_id: body.rule_id,
-                    pattern: body.pattern.clone().unwrap_or_default(),
+                    pattern: body.pattern.unwrap_or_default(),
                 }
                 .into(),
-            ));
+            );
         }
         RuleKind::_Custom(_) => {}
     }
@@ -269,7 +218,7 @@ pub async fn get_pushrule_actions_route(
 
     let mut event = db
         .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "PushRules event not found.",
@@ -279,29 +228,24 @@ pub async fn get_pushrule_actions_route(
     let actions = match body.kind {
         RuleKind::Override => global
             .override_
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.actions.clone()),
-            .map(|rule| rule.0.actions.clone()),
         RuleKind::Underride => global
             .underride
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.actions.clone()),
-            .map(|rule| rule.0.actions.clone()),
         RuleKind::Sender => global
             .sender
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.actions.clone()),
-            .map(|rule| rule.0.actions.clone()),
         RuleKind::Room => global
             .room
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.actions.clone()),
-            .map(|rule| rule.0.actions.clone()),
         RuleKind::Content => global
             .content
-            .iter()
+            .get(body.rule_id.as_str())
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .map(|rule| rule.actions.clone()),
-            .map(|rule| rule.0.actions.clone()),
         RuleKind::_Custom(_) => None,
     };
 
@@ -333,7 +277,7 @@ pub async fn set_pushrule_actions_route(
 
     let mut event = db
         .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "PushRules event not found.",
@@ -342,63 +286,33 @@ pub async fn set_pushrule_actions_route(
     let global = &mut event.content.global;
     match body.kind {
         RuleKind::Override => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.override_.get(body.rule_id.as_str()).cloned() {
-                .override_
+                rule.actions = body.actions.clone();
-                .iter()
+                global.override_.replace(rule);
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.override_.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.override_.insert(rule);
             }
         }
         RuleKind::Underride => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.underride.get(body.rule_id.as_str()).cloned() {
-                .underride
+                rule.actions = body.actions.clone();
-                .iter()
+                global.underride.replace(rule);
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.underride.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.underride.insert(rule);
             }
         }
         RuleKind::Sender => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.sender.get(body.rule_id.as_str()).cloned() {
-                .sender
+                rule.actions = body.actions.clone();
-                .iter()
+                global.sender.replace(rule);
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.sender.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.sender.insert(rule);
             }
         }
         RuleKind::Room => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.room.get(body.rule_id.as_str()).cloned() {
-                .room
+                rule.actions = body.actions.clone();
-                .iter()
+                global.room.replace(rule);
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.room.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.room.insert(rule);
             }
         }
         RuleKind::Content => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.content.get(body.rule_id.as_str()).cloned() {
-                .content
+                rule.actions = body.actions.clone();
-                .iter()
+                global.content.replace(rule);
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.content.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.content.insert(rule);
             }
         }
         RuleKind::_Custom(_) => {}
@@ -437,7 +351,7 @@ pub async fn get_pushrule_enabled_route(
 
     let mut event = db
         .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "PushRules event not found.",
@@ -448,28 +362,28 @@ pub async fn get_pushrule_enabled_route(
         RuleKind::Override => global
             .override_
             .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .find(|rule| rule.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .map_or(false, |rule| rule.enabled),
         RuleKind::Underride => global
             .underride
             .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .find(|rule| rule.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .map_or(false, |rule| rule.enabled),
         RuleKind::Sender => global
             .sender
             .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .find(|rule| rule.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .map_or(false, |rule| rule.enabled),
         RuleKind::Room => global
             .room
             .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .find(|rule| rule.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .map_or(false, |rule| rule.enabled),
         RuleKind::Content => global
             .content
             .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
+            .find(|rule| rule.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .map_or(false, |rule| rule.enabled),
         RuleKind::_Custom(_) => false,
     };
 
@@ -507,62 +421,37 @@ pub async fn set_pushrule_enabled_route(
     let global = &mut event.content.global;
     match body.kind {
         RuleKind::Override => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.override_.get(body.rule_id.as_str()).cloned() {
-                .override_
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.override_.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                 global.override_.insert(rule);
             }
         }
         RuleKind::Underride => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.underride.get(body.rule_id.as_str()).cloned() {
-                .underride
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.underride.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                 global.underride.insert(rule);
             }
         }
         RuleKind::Sender => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.sender.get(body.rule_id.as_str()).cloned() {
-                .sender
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.sender.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                 global.sender.insert(rule);
             }
         }
         RuleKind::Room => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.room.get(body.rule_id.as_str()).cloned() {
-                .room
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.room.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                 global.room.insert(rule);
             }
         }
         RuleKind::Content => {
-            if let Some(mut rule) = global
+            if let Some(mut rule) = global.content.get(body.rule_id.as_str()).cloned() {
-                .content
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.content.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                 global.content.insert(rule);
             }
         }
@@ -602,7 +491,7 @@ pub async fn delete_pushrule_route(
 
     let mut event = db
         .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "PushRules event not found.",
@@ -611,52 +500,27 @@ pub async fn delete_pushrule_route(
     let global = &mut event.content.global;
     match body.kind {
         RuleKind::Override => {
-            if let Some(rule) = global
+            if let Some(rule) = global.override_.get(body.rule_id.as_str()).cloned() {
-                .override_
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.override_.remove(&rule);
             }
         }
         RuleKind::Underride => {
-            if let Some(rule) = global
+            if let Some(rule) = global.underride.get(body.rule_id.as_str()).cloned() {
-                .underride
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.underride.remove(&rule);
             }
         }
         RuleKind::Sender => {
-            if let Some(rule) = global
+            if let Some(rule) = global.sender.get(body.rule_id.as_str()).cloned() {
-                .sender
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.sender.remove(&rule);
             }
         }
         RuleKind::Room => {
-            if let Some(rule) = global
+            if let Some(rule) = global.room.get(body.rule_id.as_str()).cloned() {
-                .room
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.room.remove(&rule);
             }
         }
         RuleKind::Content => {
-            if let Some(rule) = global
+            if let Some(rule) = global.content.get(body.rule_id.as_str()).cloned() {
-                .content
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
                 global.content.remove(&rule);
             }
         }
@@ -676,22 +540,38 @@ pub async fn delete_pushrule_route(
     Ok(delete_pushrule::Response.into())
 }
 
-#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/pushers"))]
+#[cfg_attr(
-#[tracing::instrument]
+    feature = "conduit_bin",
-pub async fn get_pushers_route() -> ConduitResult<get_pushers::Response> {
+    get("/_matrix/client/r0/pushers", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_pushers_route(
+    db: State<'_, Database>,
+    body: Ruma<get_pushers::Request>,
+) -> ConduitResult<get_pushers::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
     Ok(get_pushers::Response {
-        pushers: Vec::new(),
+        pushers: db.pusher.get_pushers(sender_user)?,
     }
     .into())
 }
 
-#[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/pushers/set"))]
+#[cfg_attr(
-#[tracing::instrument(skip(db))]
+    feature = "conduit_bin",
-pub async fn set_pushers_route(db: State<'_, Database>) -> ConduitResult<get_pushers::Response> {
+    post("/_matrix/client/r0/pushers/set", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn set_pushers_route(
+    db: State<'_, Database>,
+    body: Ruma<set_pusher::Request>,
+) -> ConduitResult<set_pusher::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+    let pusher = body.pusher.clone();
+
+    db.pusher.set_pusher(sender_user, pusher)?;
+
     db.flush().await?;
 
-    Ok(get_pushers::Response {
+    Ok(set_pusher::Response::default().into())
-        pushers: Vec::new(),
-    }
-    .into())
 }

src/client_server/read_marker.rs

@@ -47,6 +47,8 @@ pub async fn set_read_marker_route(
             ))?,
             &db.globals,
         )?;
+        db.rooms
+            .reset_notification_counts(&sender_user, &body.room_id)?;
 
         let mut user_receipts = BTreeMap::new();
         user_receipts.insert(
@@ -103,6 +105,8 @@ pub async fn create_receipt_route(
             ))?,
         &db.globals,
     )?;
+    db.rooms
+        .reset_notification_counts(&sender_user, &body.room_id)?;
 
     let mut user_receipts = BTreeMap::new();
     user_receipts.insert(

src/client_server/room.rs

@@ -1,4 +1,5 @@
 use super::State;
+use crate::client_server::invite_helper;
 use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma};
 use log::info;
 use ruma::{
@@ -91,10 +92,24 @@ pub async fn create_room_route(
     )?;
 
     // 3. Power levels
+
+    // Figure out preset. We need it for preset specific events
+    let preset = body
+        .preset
+        .clone()
+        .unwrap_or_else(|| match &body.visibility {
+            room::Visibility::Private => create_room::RoomPreset::PrivateChat,
+            room::Visibility::Public => create_room::RoomPreset::PublicChat,
+            room::Visibility::_Custom(_) => create_room::RoomPreset::PrivateChat, // Room visibility should not be custom
+        });
+
     let mut users = BTreeMap::new();
     users.insert(sender_user.clone(), 100.into());
-    for invite_ in &body.invite {
+
-        users.insert(invite_.clone(), 100.into());
+    if preset == create_room::RoomPreset::TrustedPrivateChat {
+        for invite_ in &body.invite {
+            users.insert(invite_.clone(), 100.into());
+        }
     }
 
     let power_levels_content = if let Some(power_levels) = &body.power_level_content_override {
@@ -133,16 +148,6 @@ pub async fn create_room_route(
 
     // 4. Events set by preset
 
-    // Figure out preset. We need it for preset specific events
-    let preset = body
-        .preset
-        .clone()
-        .unwrap_or_else(|| match &body.visibility {
-            room::Visibility::Private => create_room::RoomPreset::PrivateChat,
-            room::Visibility::Public => create_room::RoomPreset::PublicChat,
-            room::Visibility::_Custom(s) => create_room::RoomPreset::_Custom(s.into()),
-        });
-
     // 4.1 Join Rules
     db.rooms.build_and_append_pdu(
         PduBuilder {
@@ -265,26 +270,8 @@ pub async fn create_room_route(
     }
 
     // 7. Events implied by invite (and TODO: invite_3pid)
-    for user in &body.invite {
+    for user_id in &body.invite {
-        db.rooms.build_and_append_pdu(
+        let _ = invite_helper(sender_user, user_id, &room_id, &db, body.is_direct).await;
-            PduBuilder {
-                event_type: EventType::RoomMember,
-                content: serde_json::to_value(member::MemberEventContent {
-                    membership: member::MembershipState::Invite,
-                    displayname: db.users.displayname(&user)?,
-                    avatar_url: db.users.avatar_url(&user)?,
-                    is_direct: Some(body.is_direct),
-                    third_party_invite: None,
-                })
-                .expect("event is valid, we just created it"),
-                unsigned: None,
-                state_key: Some(user.to_string()),
-                redacts: None,
-            },
-            &sender_user,
-            &room_id,
-            &db,
-        )?;
     }
 
     // Homeserver specific stuff
@@ -343,10 +330,7 @@ pub async fn upgrade_room_route(
 ) -> ConduitResult<upgrade_room::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    if !matches!(
+    if !matches!(body.new_version, RoomVersionId::Version6) {
-        body.new_version,
-        RoomVersionId::Version5 | RoomVersionId::Version6
-    ) {
         return Err(Error::BadRequest(
             ErrorKind::UnsupportedRoomVersion,
             "This server does not support that room version.",
@@ -380,7 +364,6 @@ pub async fn upgrade_room_route(
         db.rooms
             .room_state_get(&body.room_id, &EventType::RoomCreate, "")?
             .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
-            .1
             .content,
     )
     .expect("Raw::from_value always works")
@@ -452,7 +435,7 @@ pub async fn upgrade_room_route(
     // Replicate transferable state events to the new room
     for event_type in transferable_state_events {
         let event_content = match db.rooms.room_state_get(&body.room_id, &event_type, "")? {
-            Some((_, v)) => v.content.clone(),
+            Some(v) => v.content.clone(),
             None => continue, // Skipping missing events.
         };
 
@@ -482,7 +465,6 @@ pub async fn upgrade_room_route(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")?
                 .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
-                .1
                 .content,
         )
         .expect("database contains invalid PDU")

src/client_server/session.rs

@@ -26,7 +26,12 @@ use rocket::{get, post};
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/login"))]
 #[tracing::instrument]
 pub async fn get_login_types_route() -> ConduitResult<get_login_types::Response> {
-    Ok(get_login_types::Response::new(vec![get_login_types::LoginType::Password]).into())
+    Ok(
+        get_login_types::Response::new(vec![get_login_types::LoginType::Password(
+            Default::default(),
+        )])
+        .into(),
+    )
 }
 
 /// # `POST /_matrix/client/r0/login`
@@ -51,8 +56,11 @@ pub async fn login_route(
     // Validate login method
     // TODO: Other login methods
     let user_id = match &body.login_info {
-        login::IncomingLoginInfo::Password { password } => {
+        login::IncomingLoginInfo::Password {
-            let username = if let login::IncomingUserInfo::MatrixId(matrix_id) = &body.user {
+            identifier,
+            password,
+        } => {
+            let username = if let login::IncomingUserIdentifier::MatrixId(matrix_id) = identifier {
                 matrix_id
             } else {
                 return Err(Error::BadRequest(ErrorKind::Forbidden, "Bad login type."));

src/client_server/state.rs

@@ -3,15 +3,16 @@ use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Result, Ruma};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::state::{
+        r0::state::{get_state_events, get_state_events_for_key, send_state_event},
-            get_state_events, get_state_events_for_empty_key, get_state_events_for_key,
-            send_state_event_for_empty_key, send_state_event_for_key,
-        },
     },
     events::{
-        room::history_visibility::{HistoryVisibility, HistoryVisibilityEventContent},
+        room::{
-        AnyStateEventContent, EventContent, EventType,
+            canonical_alias::CanonicalAliasEventContent,
+            history_visibility::{HistoryVisibility, HistoryVisibilityEventContent},
+        },
+        AnyStateEventContent, EventType,
     },
+    serde::Raw,
     EventId, RoomId, UserId,
 };
 
@@ -25,31 +26,23 @@ use rocket::{get, put};
 #[tracing::instrument(skip(db, body))]
 pub async fn send_state_event_for_key_route(
     db: State<'_, Database>,
-    body: Ruma<send_state_event_for_key::Request<'_>>,
+    body: Ruma<send_state_event::Request<'_>>,
-) -> ConduitResult<send_state_event_for_key::Response> {
+) -> ConduitResult<send_state_event::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    let content = serde_json::from_str::<serde_json::Value>(
-        body.json_body
-            .as_ref()
-            .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?
-            .get(),
-    )
-    .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?;
-
     let event_id = send_state_event_for_key_helper(
         &db,
         sender_user,
-        &body.content,
-        content,
         &body.room_id,
-        Some(body.state_key.to_owned()),
+        EventType::from(&body.event_type),
+        &body.body.body, // Yes, I hate it too
+        body.state_key.to_owned(),
     )
     .await?;
 
     db.flush().await?;
 
-    Ok(send_state_event_for_key::Response { event_id }.into())
+    Ok(send_state_event::Response { event_id }.into())
 }
 
 #[cfg_attr(
@@ -59,39 +52,23 @@ pub async fn send_state_event_for_key_route(
 #[tracing::instrument(skip(db, body))]
 pub async fn send_state_event_for_empty_key_route(
     db: State<'_, Database>,
-    body: Ruma<send_state_event_for_empty_key::Request<'_>>,
+    body: Ruma<send_state_event::Request<'_>>,
-) -> ConduitResult<send_state_event_for_empty_key::Response> {
+) -> ConduitResult<send_state_event::Response> {
-    // This just calls send_state_event_for_key_route
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-    let Ruma {
-        body,
-        sender_user,
-        json_body,
-        ..
-    } = body;
-
-    let json = serde_json::from_str::<serde_json::Value>(
-        json_body
-            .as_ref()
-            .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?
-            .get(),
-    )
-    .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?;
 
     let event_id = send_state_event_for_key_helper(
         &db,
-        sender_user
+        sender_user,
-            .as_ref()
-            .expect("no user for send state empty key rout"),
-        &body.content,
-        json,
         &body.room_id,
-        Some("".into()),
+        EventType::from(&body.event_type),
+        &body.body.body,
+        body.state_key.to_owned(),
     )
     .await?;
 
     db.flush().await?;
 
-    Ok(send_state_event_for_empty_key::Response { event_id }.into())
+    Ok(send_state_event::Response { event_id }.into())
 }
 
 #[cfg_attr(
@@ -112,7 +89,7 @@ pub async fn get_state_events_route(
         && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
+                .map(|event| {
                     serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
                         .map_err(|_| {
                             Error::bad_database(
@@ -159,7 +136,7 @@ pub async fn get_state_events_for_key_route(
         && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
+                .map(|event| {
                     serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
                         .map_err(|_| {
                             Error::bad_database(
@@ -183,11 +160,10 @@ pub async fn get_state_events_for_key_route(
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "State event not found.",
-        ))?
+        ))?;
-        .1;
 
     Ok(get_state_events_for_key::Response {
-        content: serde_json::value::to_raw_value(&event.content)
+        content: serde_json::from_value(event.content)
             .map_err(|_| Error::bad_database("Invalid event content in database"))?,
     }
     .into())
@@ -200,8 +176,8 @@ pub async fn get_state_events_for_key_route(
 #[tracing::instrument(skip(db, body))]
 pub async fn get_state_events_for_empty_key_route(
     db: State<'_, Database>,
-    body: Ruma<get_state_events_for_empty_key::Request<'_>>,
+    body: Ruma<get_state_events_for_key::Request<'_>>,
-) -> ConduitResult<get_state_events_for_empty_key::Response> {
+) -> ConduitResult<get_state_events_for_key::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     #[allow(clippy::blocks_in_if_conditions)]
@@ -211,7 +187,7 @@ pub async fn get_state_events_for_empty_key_route(
         && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
+                .map(|event| {
                     serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
                         .map_err(|_| {
                             Error::bad_database(
@@ -235,11 +211,10 @@ pub async fn get_state_events_for_empty_key_route(
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "State event not found.",
-        ))?
+        ))?;
-        .1;
 
-    Ok(get_state_events_for_empty_key::Response {
+    Ok(get_state_events_for_key::Response {
-        content: serde_json::value::to_raw_value(&event.content)
+        content: serde_json::from_value(event.content)
             .map_err(|_| Error::bad_database("Invalid event content in database"))?,
     }
     .into())
@@ -248,17 +223,19 @@ pub async fn get_state_events_for_empty_key_route(
 pub async fn send_state_event_for_key_helper(
     db: &Database,
     sender: &UserId,
-    content: &AnyStateEventContent,
-    json: serde_json::Value,
     room_id: &RoomId,
-    state_key: Option<String>,
+    event_type: EventType,
+    json: &Raw<AnyStateEventContent>,
+    state_key: String,
 ) -> Result<EventId> {
     let sender_user = sender;
 
-    if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content {
+    if let Ok(canonical_alias) =
+        serde_json::from_str::<CanonicalAliasEventContent>(json.json().get())
+    {
         let mut aliases = canonical_alias.alt_aliases.clone();
 
-        if let Some(alias) = canonical_alias.alias.clone() {
+        if let Some(alias) = canonical_alias.alias {
             aliases.push(alias);
         }
 
@@ -281,10 +258,10 @@ pub async fn send_state_event_for_key_helper(
 
     let event_id = db.rooms.build_and_append_pdu(
         PduBuilder {
-            event_type: content.event_type().into(),
+            event_type,
-            content: json,
+            content: serde_json::from_str(json.json().get()).expect("content is valid json"),
             unsigned: None,
-            state_key,
+            state_key: Some(state_key),
             redacts: None,
         },
         &sender_user,

src/client_server/sync.rs

@@ -11,7 +11,7 @@ use ruma::{
 use rocket::{get, tokio};
 use std::{
     collections::{hash_map, BTreeMap, HashMap, HashSet},
-    convert::TryFrom,
+    convert::{TryFrom, TryInto},
     time::Duration,
 };
 
@@ -96,7 +96,7 @@ pub async fn sync_events_route(
 
         // Database queries:
 
-        let current_state_hash = db.rooms.current_state_hash(&room_id)?;
+        let current_shortstatehash = db.rooms.current_shortstatehash(&room_id)?;
 
         // These type is Option<Option<_>>. The outer Option is None when there is no event between
         // since and the current room state, meaning there should be no updates.
@@ -109,9 +109,11 @@ pub async fn sync_events_route(
             .next()
             .is_some();
 
-        let since_state_hash = first_pdu_before_since
+        let since_shortstatehash = first_pdu_before_since.as_ref().map(|pdu| {
-            .as_ref()
+            db.rooms
-            .map(|pdu| db.rooms.pdu_state_hash(&pdu.as_ref().ok()?.0).ok()?);
+                .pdu_shortstatehash(&pdu.as_ref().ok()?.1.event_id)
+                .ok()?
+        });
 
         let (
             heroes,
@@ -119,7 +121,7 @@ pub async fn sync_events_route(
             invited_member_count,
             joined_since_last_sync,
             state_events,
-        ) = if pdus_after_since && Some(&current_state_hash) != since_state_hash.as_ref() {
+        ) = if pdus_after_since && Some(current_shortstatehash) != since_shortstatehash {
             let current_state = db.rooms.room_state_full(&room_id)?;
             let current_members = current_state
                 .iter()
@@ -129,11 +131,16 @@ pub async fn sync_events_route(
             let encrypted_room = current_state
                 .get(&(EventType::RoomEncryption, "".to_owned()))
                 .is_some();
-            let since_state = since_state_hash.as_ref().map(|state_hash| {
+            let since_state = since_shortstatehash
-                state_hash
+                .as_ref()
-                    .as_ref()
+                .map(|since_shortstatehash| {
-                    .and_then(|state_hash| db.rooms.state_full(&room_id, &state_hash).ok())
+                    Ok::<_, Error>(
-            });
+                        since_shortstatehash
+                            .map(|since_shortstatehash| db.rooms.state_full(since_shortstatehash))
+                            .transpose()?,
+                    )
+                })
+                .transpose()?;
 
             let since_encryption = since_state.as_ref().map(|state| {
                 state
@@ -315,7 +322,7 @@ pub async fn sync_events_route(
                 (None, None, Vec::new())
             };
 
-            let state_events = if dbg!(joined_since_last_sync) {
+            let state_events = if joined_since_last_sync {
                 current_state
                     .into_iter()
                     .map(|(_, pdu)| pdu.to_sync_state_event())
@@ -362,23 +369,23 @@ pub async fn sync_events_route(
         );
 
         let notification_count = if send_notification_counts {
-            if let Some(last_read) = db.rooms.edus.private_read_get(&room_id, &sender_user)? {
+            Some(
-                Some(
+                db.rooms
-                    (db.rooms
+                    .notification_count(&sender_user, &room_id)?
-                        .pdus_since(&sender_user, &room_id, last_read)?
+                    .try_into()
-                        .filter_map(|pdu| pdu.ok()) // Filter out buggy events
+                    .expect("notification count can't go that high"),
-                        .filter(|(_, pdu)| {
+            )
-                            matches!(
+        } else {
-                                pdu.kind.clone(),
+            None
-                                EventType::RoomMessage | EventType::RoomEncrypted
+        };
-                            )
+
-                        })
+        let highlight_count = if send_notification_counts {
-                        .count() as u32)
+            Some(
-                        .into(),
+                db.rooms
-                )
+                    .highlight_count(&sender_user, &room_id)?
-            } else {
+                    .try_into()
-                None
+                    .expect("highlight count can't go that high"),
-            }
+            )
         } else {
             None
         };
@@ -432,7 +439,7 @@ pub async fn sync_events_route(
                 invited_member_count: invited_member_count.map(|n| (n as u32).into()),
             },
             unread_notifications: sync_events::UnreadNotificationsCount {
-                highlight_count: None,
+                highlight_count,
                 notification_count,
             },
             timeline: sync_events::Timeline {
@@ -486,85 +493,17 @@ pub async fn sync_events_route(
     }
 
     let mut left_rooms = BTreeMap::new();
-    for room_id in db.rooms.rooms_left(&sender_user) {
+    for result in db.rooms.rooms_left(&sender_user) {
-        let room_id = room_id?;
+        let (room_id, left_state_events) = result?;
+        let left_count = db.rooms.get_left_count(&room_id, &sender_user)?;
 
-        let since_member = if let Some(since_member) = db
+        // Left before last sync
-            .rooms
+        if Some(since) >= left_count {
-            .pdus_after(sender_user, &room_id, since)
-            .next()
-            .and_then(|pdu| pdu.ok())
-            .and_then(|pdu| {
-                db.rooms
-                    .pdu_state_hash(&pdu.0)
-                    .ok()?
-                    .ok_or_else(|| Error::bad_database("Pdu in db doesn't have a state hash."))
-                    .ok()
-            })
-            .and_then(|state_hash| {
-                db.rooms
-                    .state_get(
-                        &room_id,
-                        &state_hash,
-                        &EventType::RoomMember,
-                        sender_user.as_str(),
-                    )
-                    .ok()?
-                    .ok_or_else(|| Error::bad_database("State hash in db doesn't have a state."))
-                    .ok()
-            })
-            .and_then(|(pdu_id, pdu)| {
-                serde_json::from_value::<Raw<ruma::events::room::member::MemberEventContent>>(
-                    pdu.content.clone(),
-                )
-                .expect("Raw::from_value always works")
-                .deserialize()
-                .map_err(|_| Error::bad_database("Invalid PDU in database."))
-                .map(|content| (pdu_id, pdu, content))
-                .ok()
-            }) {
-            since_member
-        } else {
-            // We couldn't find the since_member event. This is very weird - we better abort
             continue;
-        };
+        }
 
-        let left_since_last_sync = since_member.2.membership == MembershipState::Join;
+        left_rooms.insert(
-
+            room_id.clone(),
-        let left_room = if left_since_last_sync {
-            device_list_left.extend(
-                db.rooms
-                    .room_members(&room_id)
-                    .filter_map(|user_id| Some(user_id.ok()?))
-                    .filter(|user_id| {
-                        // Don't send key updates from the sender to the sender
-                        sender_user != user_id
-                    })
-                    .filter(|user_id| {
-                        // Only send if the sender doesn't share any encrypted room with the target
-                        // anymore
-                        !share_encrypted_room(&db, sender_user, user_id, &room_id)
-                    }),
-            );
-
-            let pdus = db.rooms.pdus_since(&sender_user, &room_id, since)?;
-            let mut room_events = pdus
-                .filter_map(|pdu| pdu.ok()) // Filter out buggy events
-                .take_while(|(pdu_id, _)| since_member.0 != pdu_id)
-                .map(|(_, pdu)| pdu.to_sync_room_event())
-                .collect::<Vec<_>>();
-            room_events.push(since_member.1.to_sync_room_event());
-
-            sync_events::LeftRoom {
-                account_data: sync_events::AccountData { events: Vec::new() },
-                timeline: sync_events::Timeline {
-                    limited: false,
-                    prev_batch: Some(next_batch.clone()),
-                    events: room_events,
-                },
-                state: sync_events::State { events: Vec::new() },
-            }
-        } else {
             sync_events::LeftRoom {
                 account_data: sync_events::AccountData { events: Vec::new() },
                 timeline: sync_events::Timeline {
@@ -572,54 +511,31 @@ pub async fn sync_events_route(
                     prev_batch: Some(next_batch.clone()),
                     events: Vec::new(),
                 },
-                state: sync_events::State { events: Vec::new() },
+                state: sync_events::State {
-            }
+                    events: left_state_events,
-        };
+                },
-
+            },
-        if !left_room.is_empty() {
+        );
-            left_rooms.insert(room_id.clone(), left_room);
-        }
     }
 
     let mut invited_rooms = BTreeMap::new();
-    for room_id in db.rooms.rooms_invited(&sender_user) {
+    for result in db.rooms.rooms_invited(&sender_user) {
-        let room_id = room_id?;
+        let (room_id, invite_state_events) = result?;
-        let mut invited_since_last_sync = false;
+        let invite_count = db.rooms.get_invite_count(&room_id, &sender_user)?;
-        for pdu in db.rooms.pdus_since(&sender_user, &room_id, since)? {
-            let (_, pdu) = pdu?;
-            if pdu.kind == EventType::RoomMember && pdu.state_key == Some(sender_user.to_string()) {
-                let content = serde_json::from_value::<
-                    Raw<ruma::events::room::member::MemberEventContent>,
-                >(pdu.content.clone())
-                .expect("Raw::from_value always works")
-                .deserialize()
-                .map_err(|_| Error::bad_database("Invalid PDU in database."))?;
 
-                if content.membership == MembershipState::Invite {
+        // Invited before last sync
-                    invited_since_last_sync = true;
+        if Some(since) >= invite_count {
-                    break;
-                }
-            }
-        }
-
-        if !invited_since_last_sync {
             continue;
         }
 
-        let invited_room = sync_events::InvitedRoom {
+        invited_rooms.insert(
-            invite_state: sync_events::InviteState {
+            room_id.clone(),
-                events: db
+            sync_events::InvitedRoom {
-                    .rooms
+                invite_state: sync_events::InviteState {
-                    .room_state_full(&room_id)?
+                    events: invite_state_events,
-                    .into_iter()
+                },
-                    .map(|(_, pdu)| pdu.to_stripped_state_event())
-                    .collect(),
             },
-        };
+        );
-
-        if !invited_room.is_empty() {
-            invited_rooms.insert(room_id.clone(), invited_room);
-        }
     }
 
     for user_id in left_encrypted_users {
@@ -703,12 +619,7 @@ pub async fn sync_events_route(
         if duration.as_secs() > 30 {
             duration = Duration::from_secs(30);
         }
-        let delay = tokio::time::sleep(duration);
+        let _ = tokio::time::timeout(duration, watcher).await;
-        tokio::pin!(delay);
-        tokio::select! {
-            _ = &mut delay => {}
-            _ = watcher => {}
-        }
     }
 
     Ok(response.into())

src/client_server/thirdparty.rs

@@ -1,7 +1,6 @@
 use crate::ConduitResult;
 use ruma::api::client::r0::thirdparty::get_protocols;
 
-use log::warn;
 #[cfg(feature = "conduit_bin")]
 use rocket::get;
 use std::collections::BTreeMap;
@@ -12,7 +11,7 @@ use std::collections::BTreeMap;
 )]
 #[tracing::instrument]
 pub async fn get_protocols_route() -> ConduitResult<get_protocols::Response> {
-    warn!("TODO: get_protocols_route");
+    // TODO
     Ok(get_protocols::Response {
         protocols: BTreeMap::new(),
     }

src/database.rs

@@ -4,6 +4,7 @@ pub mod appservice;
 pub mod globals;
 pub mod key_backups;
 pub mod media;
+pub mod pusher;
 pub mod rooms;
 pub mod sending;
 pub mod transaction_ids;
@@ -45,6 +46,8 @@ pub struct Config {
     jwt_secret: Option<String>,
     #[serde(default = "Vec::new")]
     trusted_servers: Vec<Box<ServerName>>,
+    #[serde(default = "default_log")]
+    pub log: String,
 }
 
 fn false_fn() -> bool {
@@ -67,6 +70,10 @@ fn default_max_concurrent_requests() -> u16 {
     4
 }
 
+fn default_log() -> String {
+    "info,state_res=warn,rocket=off,_=off,sled=off".to_owned()
+}
+
 #[derive(Clone)]
 pub struct Database {
     pub globals: globals::Globals,
@@ -80,6 +87,7 @@ pub struct Database {
     pub sending: sending::Sending,
     pub admin: admin::Admin,
     pub appservice: appservice::Appservice,
+    pub pusher: pusher::PushData,
     pub _db: sled::Db,
 }
 
@@ -101,6 +109,7 @@ impl Database {
         let db = sled::Config::default()
             .path(&config.database_path)
             .cache_capacity(config.cache_capacity as u64)
+            .use_compression(true)
             .open()?;
 
         info!("Opened sled database at {}", config.database_path);
@@ -108,21 +117,17 @@ impl Database {
         let (admin_sender, admin_receiver) = mpsc::unbounded();
 
         let db = Self {
-            globals: globals::Globals::load(
-                db.open_tree("global")?,
-                db.open_tree("servertimeout_signingkey")?,
-                config,
-            )?,
             users: users::Users {
                 userid_password: db.open_tree("userid_password")?,
                 userid_displayname: db.open_tree("userid_displayname")?,
                 userid_avatarurl: db.open_tree("userid_avatarurl")?,
                 userdeviceid_token: db.open_tree("userdeviceid_token")?,
                 userdeviceid_metadata: db.open_tree("userdeviceid_metadata")?,
+                userid_devicelistversion: db.open_tree("userid_devicelistversion")?,
                 token_userdeviceid: db.open_tree("token_userdeviceid")?,
                 onetimekeyid_onetimekeys: db.open_tree("onetimekeyid_onetimekeys")?,
                 userid_lastonetimekeyupdate: db.open_tree("userid_lastonetimekeyupdate")?,
-                keychangeid_userid: db.open_tree("devicekeychangeid_userid")?,
+                keychangeid_userid: db.open_tree("keychangeid_userid")?,
                 keyid_key: db.open_tree("keyid_key")?,
                 userid_masterkeyid: db.open_tree("userid_masterkeyid")?,
                 userid_selfsigningkeyid: db.open_tree("userid_selfsigningkeyid")?,
@@ -137,7 +142,7 @@ impl Database {
                     readreceiptid_readreceipt: db.open_tree("readreceiptid_readreceipt")?,
                     roomuserid_privateread: db.open_tree("roomuserid_privateread")?, // "Private" read receipt
                     roomuserid_lastprivatereadupdate: db
-                        .open_tree("roomid_lastprivatereadupdate")?,
+                        .open_tree("roomuserid_lastprivatereadupdate")?,
                     typingid_userid: db.open_tree("typingid_userid")?,
                     roomid_lasttypingupdate: db.open_tree("roomid_lasttypingupdate")?,
                     presenceid_presence: db.open_tree("presenceid_presence")?,
@@ -148,7 +153,7 @@ impl Database {
                 roomid_pduleaves: db.open_tree("roomid_pduleaves")?,
 
                 alias_roomid: db.open_tree("alias_roomid")?,
-                aliasid_alias: db.open_tree("alias_roomid")?,
+                aliasid_alias: db.open_tree("aliasid_alias")?,
                 publicroomids: db.open_tree("publicroomids")?,
 
                 tokenids: db.open_tree("tokenids")?,
@@ -157,15 +162,23 @@ impl Database {
                 userroomid_joined: db.open_tree("userroomid_joined")?,
                 roomuserid_joined: db.open_tree("roomuserid_joined")?,
                 roomuseroncejoinedids: db.open_tree("roomuseroncejoinedids")?,
-                userroomid_invited: db.open_tree("userroomid_invited")?,
+                userroomid_invitestate: db.open_tree("userroomid_invitestate")?,
-                roomuserid_invited: db.open_tree("roomuserid_invited")?,
+                roomuserid_invitecount: db.open_tree("roomuserid_invitecount")?,
-                userroomid_left: db.open_tree("userroomid_left")?,
+                userroomid_leftstate: db.open_tree("userroomid_leftstate")?,
+                roomuserid_leftcount: db.open_tree("roomuserid_leftcount")?,
 
-                statekey_short: db.open_tree("statekey_short")?,
+                userroomid_notificationcount: db.open_tree("userroomid_notificationcount")?,
-                stateid_pduid: db.open_tree("stateid_pduid")?,
+                userroomid_highlightcount: db.open_tree("userroomid_highlightcount")?,
-                pduid_statehash: db.open_tree("pduid_statehash")?,
+
-                roomid_statehash: db.open_tree("roomid_statehash")?,
+                statekey_shortstatekey: db.open_tree("statekey_shortstatekey")?,
-                eventid_outlierpdu: db.open_tree("roomeventid_outlierpdu")?,
+                stateid_shorteventid: db.open_tree("stateid_shorteventid")?,
+                eventid_shorteventid: db.open_tree("eventid_shorteventid")?,
+                shorteventid_eventid: db.open_tree("shorteventid_eventid")?,
+                shorteventid_shortstatehash: db.open_tree("shorteventid_shortstatehash")?,
+                roomid_shortstatehash: db.open_tree("roomid_shortstatehash")?,
+                statehash_shortstatehash: db.open_tree("statehash_shortstatehash")?,
+
+                eventid_outlierpdu: db.open_tree("eventid_outlierpdu")?,
                 prevevent_parent: db.open_tree("prevevent_parent")?,
             },
             account_data: account_data::AccountData {
@@ -177,7 +190,7 @@ impl Database {
             key_backups: key_backups::KeyBackups {
                 backupid_algorithm: db.open_tree("backupid_algorithm")?,
                 backupid_etag: db.open_tree("backupid_etag")?,
-                backupkeyid_backup: db.open_tree("backupkeyid_backupmetadata")?,
+                backupkeyid_backup: db.open_tree("backupkeyid_backup")?,
             },
             transaction_ids: transaction_ids::TransactionIds {
                 userdevicetxnid_response: db.open_tree("userdevicetxnid_response")?,
@@ -185,7 +198,7 @@ impl Database {
             sending: sending::Sending {
                 servernamepduids: db.open_tree("servernamepduids")?,
                 servercurrentpdus: db.open_tree("servercurrentpdus")?,
-                maximum_requests: Arc::new(Semaphore::new(10)),
+                maximum_requests: Arc::new(Semaphore::new(config.max_concurrent_requests as usize)),
             },
             admin: admin::Admin {
                 sender: admin_sender,
@@ -194,6 +207,12 @@ impl Database {
                 cached_registrations: Arc::new(RwLock::new(HashMap::new())),
                 id_appserviceregistrations: db.open_tree("id_appserviceregistrations")?,
             },
+            pusher: pusher::PushData::new(&db)?,
+            globals: globals::Globals::load(
+                db.open_tree("global")?,
+                db.open_tree("servertimeout_signingkey")?,
+                config,
+            )?,
             _db: db,
         };
 
@@ -203,7 +222,7 @@ impl Database {
     }
 
     pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) {
-        let userid_bytes = user_id.to_string().as_bytes().to_vec();
+        let userid_bytes = user_id.as_bytes().to_vec();
         let mut userid_prefix = userid_bytes.clone();
         userid_prefix.push(0xff);
 
@@ -222,12 +241,16 @@ impl Database {
         );
 
         futures.push(self.rooms.userroomid_joined.watch_prefix(&userid_prefix));
-        futures.push(self.rooms.userroomid_invited.watch_prefix(&userid_prefix));
+        futures.push(
-        futures.push(self.rooms.userroomid_left.watch_prefix(&userid_prefix));
+            self.rooms
+                .userroomid_invitestate
+                .watch_prefix(&userid_prefix),
+        );
+        futures.push(self.rooms.userroomid_leftstate.watch_prefix(&userid_prefix));
 
         // Events for rooms we are in
         for room_id in self.rooms.rooms_joined(user_id).filter_map(|r| r.ok()) {
-            let roomid_bytes = room_id.to_string().as_bytes().to_vec();
+            let roomid_bytes = room_id.as_bytes().to_vec();
             let mut roomid_prefix = roomid_bytes.clone();
             roomid_prefix.push(0xff);
 
@@ -287,7 +310,8 @@ impl Database {
     }
 
     pub async fn flush(&self) -> Result<()> {
-        self._db.flush_async().await?;
+        // noop while we don't use sled 1.0
+        //self._db.flush_async().await?;
         Ok(())
     }
 }

src/database/account_data.rs

@@ -30,7 +30,7 @@ impl AccountData {
             .as_bytes()
             .to_vec();
         prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
+        prefix.extend_from_slice(&user_id.as_bytes());
         prefix.push(0xff);
 
         // Remove old entry
@@ -42,7 +42,7 @@ impl AccountData {
         let mut key = prefix;
         key.extend_from_slice(&globals.next_count()?.to_be_bytes());
         key.push(0xff);
-        key.extend_from_slice(event_type.to_string().as_bytes());
+        key.extend_from_slice(event_type.as_ref().as_bytes());
 
         let json = serde_json::to_value(data).expect("all types here can be serialized"); // TODO: maybe add error handling
         if json.get("type").is_none() || json.get("content").is_none() {
@@ -89,7 +89,7 @@ impl AccountData {
             .as_bytes()
             .to_vec();
         prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
+        prefix.extend_from_slice(&user_id.as_bytes());
         prefix.push(0xff);
 
         // Skip the data that's exactly at since, because we sent that last time
@@ -135,7 +135,7 @@ impl AccountData {
             .as_bytes()
             .to_vec();
         prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
+        prefix.extend_from_slice(&user_id.as_bytes());
         prefix.push(0xff);
         let kind = kind.clone();
 
@@ -148,7 +148,7 @@ impl AccountData {
                         k.rsplit(|&b| b == 0xff)
                             .next()
                             .map(|current_event_type| {
-                                current_event_type == kind.to_string().as_bytes()
+                                current_event_type == kind.as_ref().as_bytes()
                             })
                             .unwrap_or(false)
                     })

src/database/appservice.rs

@@ -55,9 +55,7 @@ impl Appservice {
         })
     }
 
-    pub fn iter_all<'a>(
+    pub fn iter_all(&self) -> impl Iterator<Item = Result<(String, serde_yaml::Value)>> + '_ {
-        &'a self,
-    ) -> impl Iterator<Item = Result<(String, serde_yaml::Value)>> + 'a {
         self.iter_ids().filter_map(|id| id.ok()).map(move |id| {
             Ok((
                 id.clone(),

src/database/globals.rs

@@ -1,9 +1,10 @@
 use crate::{database::Config, utils, Error, Result};
-use log::error;
+use log::{error, info};
 use ruma::{
     api::federation::discovery::{ServerSigningKeys, VerifyKey},
     ServerName, ServerSigningKeyId,
 };
+use rustls::{ServerCertVerifier, WebPKIVerifier};
 use std::{
     collections::{BTreeMap, HashMap},
     sync::{Arc, RwLock},
@@ -13,11 +14,12 @@ use trust_dns_resolver::TokioAsyncResolver;
 
 pub const COUNTER: &str = "c";
 
-type WellKnownMap = HashMap<Box<ServerName>, (String, Option<String>)>;
+type WellKnownMap = HashMap<Box<ServerName>, (String, String)>;
-
+type TlsNameMap = HashMap<String, webpki::DNSName>;
 #[derive(Clone)]
 pub struct Globals {
     pub actual_destination_cache: Arc<RwLock<WellKnownMap>>, // actual_destination, host
+    pub tls_name_override: Arc<RwLock<TlsNameMap>>,
     pub(super) globals: sled::Tree,
     config: Config,
     keypair: Arc<ruma::signatures::Ed25519KeyPair>,
@@ -27,6 +29,39 @@ pub struct Globals {
     pub(super) servertimeout_signingkey: sled::Tree, // ServerName + Timeout Timestamp -> algorithm:key + pubkey
 }
 
+struct MatrixServerVerifier {
+    inner: WebPKIVerifier,
+    tls_name_override: Arc<RwLock<TlsNameMap>>,
+}
+
+impl ServerCertVerifier for MatrixServerVerifier {
+    fn verify_server_cert(
+        &self,
+        roots: &rustls::RootCertStore,
+        presented_certs: &[rustls::Certificate],
+        dns_name: webpki::DNSNameRef<'_>,
+        ocsp_response: &[u8],
+    ) -> std::result::Result<rustls::ServerCertVerified, rustls::TLSError> {
+        if let Some(override_name) = self.tls_name_override.read().unwrap().get(dns_name.into()) {
+            let result = self.inner.verify_server_cert(
+                roots,
+                presented_certs,
+                override_name.as_ref(),
+                ocsp_response,
+            );
+            if result.is_ok() {
+                return result;
+            }
+            info!(
+                "Server {:?} is non-compliant, retrying TLS verification with original name",
+                dns_name
+            );
+        }
+        self.inner
+            .verify_server_cert(roots, presented_certs, dns_name, ocsp_response)
+    }
+}
+
 impl Globals {
     pub fn load(
         globals: sled::Tree,
@@ -67,10 +102,21 @@ impl Globals {
             }
         };
 
+        let tls_name_override = Arc::new(RwLock::new(TlsNameMap::new()));
+        let verifier = Arc::new(MatrixServerVerifier {
+            inner: WebPKIVerifier::new(),
+            tls_name_override: tls_name_override.clone(),
+        });
+        let mut tlsconfig = rustls::ClientConfig::new();
+        tlsconfig.dangerous().set_certificate_verifier(verifier);
+        tlsconfig.root_store =
+            rustls_native_certs::load_native_certs().expect("Error loading system certificates");
+
         let reqwest_client = reqwest::Client::builder()
             .connect_timeout(Duration::from_secs(30))
             .timeout(Duration::from_secs(60 * 3))
             .pool_max_idle_per_host(1)
+            .use_preconfigured_tls(tlsconfig)
             .build()
             .unwrap();
 
@@ -87,9 +133,10 @@ impl Globals {
             dns_resolver: TokioAsyncResolver::tokio_from_system_conf().map_err(|_| {
                 Error::bad_config("Failed to set up trust dns resolver with system config.")
             })?,
-            actual_destination_cache: Arc::new(RwLock::new(HashMap::new())),
+            actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())),
-            jwt_decoding_key,
+            tls_name_override,
             servertimeout_signingkey,
+            jwt_decoding_key,
         })
     }
 

src/database/key_backups.rs

@@ -24,7 +24,7 @@ impl KeyBackups {
     ) -> Result<String> {
         let version = globals.next_count()?.to_string();
 
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
 
@@ -39,7 +39,7 @@ impl KeyBackups {
     }
 
     pub fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
 
@@ -67,7 +67,7 @@ impl KeyBackups {
         backup_metadata: &BackupAlgorithm,
         globals: &super::globals::Globals,
     ) -> Result<String> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
 
@@ -89,7 +89,7 @@ impl KeyBackups {
     }
 
     pub fn get_latest_backup(&self, user_id: &UserId) -> Result<Option<(String, BackupAlgorithm)>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         self.backupid_algorithm
             .scan_prefix(&prefix)
@@ -113,7 +113,7 @@ impl KeyBackups {
     }
 
     pub fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Option<BackupAlgorithm>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(version.as_bytes());
 
@@ -132,7 +132,7 @@ impl KeyBackups {
         key_data: &KeyBackupData,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(version.as_bytes());
 
@@ -147,7 +147,7 @@ impl KeyBackups {
             .insert(&key, &globals.next_count()?.to_be_bytes())?;
 
         key.push(0xff);
-        key.extend_from_slice(room_id.to_string().as_bytes());
+        key.extend_from_slice(room_id.as_bytes());
         key.push(0xff);
         key.extend_from_slice(session_id.as_bytes());
 
@@ -160,7 +160,7 @@ impl KeyBackups {
     }
 
     pub fn count_keys(&self, user_id: &UserId, version: &str) -> Result<usize> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
 
@@ -168,7 +168,7 @@ impl KeyBackups {
     }
 
     pub fn get_etag(&self, user_id: &UserId, version: &str) -> Result<String> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
 
@@ -187,7 +187,7 @@ impl KeyBackups {
         user_id: &UserId,
         version: &str,
     ) -> Result<BTreeMap<RoomId, RoomKeyBackup>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
         prefix.push(0xff);
@@ -240,7 +240,7 @@ impl KeyBackups {
         version: &str,
         room_id: &RoomId,
     ) -> BTreeMap<String, KeyBackupData> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
         prefix.push(0xff);
@@ -278,7 +278,7 @@ impl KeyBackups {
         room_id: &RoomId,
         session_id: &str,
     ) -> Result<Option<KeyBackupData>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(version.as_bytes());
         key.push(0xff);
@@ -297,7 +297,7 @@ impl KeyBackups {
     }
 
     pub fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
         key.push(0xff);
@@ -320,7 +320,7 @@ impl KeyBackups {
         version: &str,
         room_id: &RoomId,
     ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
         key.push(0xff);
@@ -346,7 +346,7 @@ impl KeyBackups {
         room_id: &RoomId,
         session_id: &str,
     ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
         key.push(0xff);

src/database/media.rs

@@ -226,16 +226,17 @@ impl Media {
                 }
 
                 let thumbnail = if crop {
-                    image.resize_to_fill(width, height, FilterType::Triangle)
+                    image.resize_to_fill(width, height, FilterType::CatmullRom)
                 } else {
                     let (exact_width, exact_height) = {
                         // Copied from image::dynimage::resize_dimensions
                         let ratio = u64::from(original_width) * u64::from(height);
                         let nratio = u64::from(width) * u64::from(original_height);
 
-                        let use_width = nratio > ratio;
+                        let use_width = nratio <= ratio;
                         let intermediate = if use_width {
-                            u64::from(original_height) * u64::from(width) / u64::from(width)
+                            u64::from(original_height) * u64::from(width)
+                                / u64::from(original_width)
                         } else {
                             u64::from(original_width) * u64::from(height)
                                 / u64::from(original_height)

src/database/pusher.rs

@@ -0,0 +1,327 @@
+use crate::{Database, Error, PduEvent, Result};
+use bytes::BytesMut;
+use log::{error, info, warn};
+use ruma::{
+    api::{
+        client::r0::push::{get_pushers, set_pusher, PusherKind},
+        push_gateway::send_event_notification::{
+            self,
+            v1::{Device, Notification, NotificationCounts, NotificationPriority},
+        },
+        IncomingResponse, OutgoingRequest, SendAccessToken,
+    },
+    events::{room::power_levels::PowerLevelsEventContent, EventType},
+    push::{Action, PushConditionRoomCtx, PushFormat, Ruleset, Tweak},
+    uint, UInt, UserId,
+};
+use sled::IVec;
+
+use std::{convert::TryFrom, fmt::Debug};
+
+#[derive(Debug, Clone)]
+pub struct PushData {
+    /// UserId + pushkey -> Pusher
+    pub(super) senderkey_pusher: sled::Tree,
+}
+
+impl PushData {
+    pub fn new(db: &sled::Db) -> Result<Self> {
+        Ok(Self {
+            senderkey_pusher: db.open_tree("senderkey_pusher")?,
+        })
+    }
+
+    pub fn set_pusher(&self, sender: &UserId, pusher: set_pusher::Pusher) -> Result<()> {
+        let mut key = sender.as_bytes().to_vec();
+        key.push(0xff);
+        key.extend_from_slice(pusher.pushkey.as_bytes());
+
+        // There are 2 kinds of pushers but the spec says: null deletes the pusher.
+        if pusher.kind.is_none() {
+            return self
+                .senderkey_pusher
+                .remove(key)
+                .map(|_| ())
+                .map_err(Into::into);
+        }
+
+        self.senderkey_pusher.insert(
+            key,
+            &*serde_json::to_string(&pusher).expect("Pusher is valid JSON string"),
+        )?;
+
+        Ok(())
+    }
+
+    pub fn get_pusher(&self, senderkey: &[u8]) -> Result<Option<get_pushers::Pusher>> {
+        self.senderkey_pusher
+            .get(senderkey)?
+            .map(|push| {
+                Ok(serde_json::from_slice(&*push)
+                    .map_err(|_| Error::bad_database("Invalid Pusher in db."))?)
+            })
+            .transpose()
+    }
+
+    pub fn get_pushers(&self, sender: &UserId) -> Result<Vec<get_pushers::Pusher>> {
+        let mut prefix = sender.as_bytes().to_vec();
+        prefix.push(0xff);
+
+        self.senderkey_pusher
+            .scan_prefix(prefix)
+            .values()
+            .map(|push| {
+                let push = push.map_err(|_| Error::bad_database("Invalid push bytes in db."))?;
+                Ok(serde_json::from_slice(&*push)
+                    .map_err(|_| Error::bad_database("Invalid Pusher in db."))?)
+            })
+            .collect()
+    }
+
+    pub fn get_pusher_senderkeys(&self, sender: &UserId) -> impl Iterator<Item = Result<IVec>> {
+        let mut prefix = sender.as_bytes().to_vec();
+        prefix.push(0xff);
+
+        self.senderkey_pusher
+            .scan_prefix(prefix)
+            .keys()
+            .map(|r| Ok(r?))
+    }
+}
+
+pub async fn send_request<T: OutgoingRequest>(
+    globals: &crate::database::globals::Globals,
+    destination: &str,
+    request: T,
+) -> Result<T::IncomingResponse>
+where
+    T: Debug,
+{
+    let destination = destination.replace("/_matrix/push/v1/notify", "");
+
+    let http_request = request
+        .try_into_http_request::<BytesMut>(&destination, SendAccessToken::IfRequired(""))
+        .map_err(|e| {
+            warn!("Failed to find destination {}: {}", destination, e);
+            Error::BadServerResponse("Invalid destination")
+        })?
+        .map(|body| body.freeze());
+
+    let reqwest_request = reqwest::Request::try_from(http_request)
+        .expect("all http requests are valid reqwest requests");
+
+    // TODO: we could keep this very short and let expo backoff do it's thing...
+    //*reqwest_request.timeout_mut() = Some(Duration::from_secs(5));
+
+    let url = reqwest_request.url().clone();
+    let reqwest_response = globals.reqwest_client().execute(reqwest_request).await;
+
+    // Because reqwest::Response -> http::Response is complicated:
+    match reqwest_response {
+        Ok(mut reqwest_response) => {
+            let status = reqwest_response.status();
+            let mut http_response = http::Response::builder().status(status);
+            let headers = http_response.headers_mut().unwrap();
+
+            for (k, v) in reqwest_response.headers_mut().drain() {
+                if let Some(key) = k {
+                    headers.insert(key, v);
+                }
+            }
+
+            let status = reqwest_response.status();
+
+            let body = reqwest_response.bytes().await.unwrap_or_else(|e| {
+                warn!("server error {}", e);
+                Vec::new().into()
+            }); // TODO: handle timeout
+
+            if status != 200 {
+                info!(
+                    "Push gateway returned bad response {} {}\n{}\n{:?}",
+                    destination,
+                    status,
+                    url,
+                    crate::utils::string_from_bytes(&body)
+                );
+            }
+
+            let response = T::IncomingResponse::try_from_http_response(
+                http_response
+                    .body(body)
+                    .expect("reqwest body is valid http body"),
+            );
+            response.map_err(|_| {
+                info!(
+                    "Push gateway returned invalid response bytes {}\n{}",
+                    destination, url
+                );
+                Error::BadServerResponse("Push gateway returned bad response.")
+            })
+        }
+        Err(e) => Err(e.into()),
+    }
+}
+
+pub async fn send_push_notice(
+    user: &UserId,
+    unread: UInt,
+    pusher: &get_pushers::Pusher,
+    ruleset: Ruleset,
+    pdu: &PduEvent,
+    db: &Database,
+) -> Result<()> {
+    let mut notify = None;
+    let mut tweaks = Vec::new();
+
+    for action in get_actions(user, &ruleset, pdu, db)? {
+        let n = match action {
+            Action::DontNotify => false,
+            // TODO: Implement proper support for coalesce
+            Action::Notify | Action::Coalesce => true,
+            Action::SetTweak(tweak) => {
+                tweaks.push(tweak.clone());
+                continue;
+            }
+        };
+
+        if notify.is_some() {
+            return Err(Error::bad_database(
+                r#"Malformed pushrule contains more than one of these actions: ["dont_notify", "notify", "coalesce"]"#,
+            ));
+        }
+
+        notify = Some(n);
+    }
+
+    if notify == Some(true) {
+        send_notice(unread, pusher, tweaks, pdu, db).await?;
+    }
+    // Else the event triggered no actions
+
+    Ok(())
+}
+
+pub fn get_actions<'a>(
+    user: &UserId,
+    ruleset: &'a Ruleset,
+    pdu: &PduEvent,
+    db: &Database,
+) -> Result<&'a [Action]> {
+    let power_levels: PowerLevelsEventContent = db
+        .rooms
+        .room_state_get(&pdu.room_id, &EventType::RoomPowerLevels, "")?
+        .map(|ev| {
+            serde_json::from_value(ev.content)
+                .map_err(|_| Error::bad_database("invalid m.room.power_levels event"))
+        })
+        .transpose()?
+        .unwrap_or_default();
+
+    let ctx = PushConditionRoomCtx {
+        room_id: pdu.room_id.clone(),
+        member_count: 10_u32.into(), // TODO: get member count efficiently
+        user_display_name: db
+            .users
+            .displayname(&user)?
+            .unwrap_or_else(|| user.localpart().to_owned()),
+        users_power_levels: power_levels.users,
+        default_power_level: power_levels.users_default,
+        notification_power_levels: power_levels.notifications,
+    };
+
+    Ok(ruleset.get_actions(&pdu.to_sync_room_event(), &ctx))
+}
+
+async fn send_notice(
+    unread: UInt,
+    pusher: &get_pushers::Pusher,
+    tweaks: Vec<Tweak>,
+    event: &PduEvent,
+    db: &Database,
+) -> Result<()> {
+    // TODO: email
+    if pusher.kind == PusherKind::Email {
+        return Ok(());
+    }
+
+    // TODO:
+    // Two problems with this
+    // 1. if "event_id_only" is the only format kind it seems we should never add more info
+    // 2. can pusher/devices have conflicting formats
+    let event_id_only = pusher.data.format == Some(PushFormat::EventIdOnly);
+    let url = if let Some(url) = &pusher.data.url {
+        url
+    } else {
+        error!("Http Pusher must have URL specified.");
+        return Ok(());
+    };
+
+    let mut device = Device::new(pusher.app_id.clone(), pusher.pushkey.clone());
+    let mut data_minus_url = pusher.data.clone();
+    // The url must be stripped off according to spec
+    data_minus_url.url = None;
+    device.data = Some(data_minus_url);
+
+    // Tweaks are only added if the format is NOT event_id_only
+    if !event_id_only {
+        device.tweaks = tweaks.clone();
+    }
+
+    let d = &[device];
+    let mut notifi = Notification::new(d);
+
+    notifi.prio = NotificationPriority::Low;
+    notifi.event_id = Some(&event.event_id);
+    notifi.room_id = Some(&event.room_id);
+    // TODO: missed calls
+    notifi.counts = NotificationCounts::new(unread, uint!(0));
+
+    if event.kind == EventType::RoomEncrypted
+        || tweaks
+            .iter()
+            .any(|t| matches!(t, Tweak::Highlight(true) | Tweak::Sound(_)))
+    {
+        notifi.prio = NotificationPriority::High
+    }
+
+    if event_id_only {
+        send_request(
+            &db.globals,
+            &url,
+            send_event_notification::v1::Request::new(notifi),
+        )
+        .await?;
+    } else {
+        notifi.sender = Some(&event.sender);
+        notifi.event_type = Some(&event.kind);
+        notifi.content = serde_json::value::to_raw_value(&event.content).ok();
+
+        if event.kind == EventType::RoomMember {
+            notifi.user_is_target = event.state_key.as_deref() == Some(event.sender.as_str());
+        }
+
+        let user_name = db.users.displayname(&event.sender)?;
+        notifi.sender_display_name = user_name.as_deref();
+        let room_name = db
+            .rooms
+            .room_state_get(&event.room_id, &EventType::RoomName, "")?
+            .map(|pdu| match pdu.content.get("name") {
+                Some(serde_json::Value::String(s)) => Some(s.to_string()),
+                _ => None,
+            })
+            .flatten();
+        notifi.room_name = room_name.as_deref();
+
+        send_request(
+            &db.globals,
+            &url,
+            send_event_notification::v1::Request::new(notifi),
+        )
+        .await?;
+    }
+
+    // TODO: email
+
+    Ok(())
+}

src/database/rooms/edus.rs

@@ -9,7 +9,7 @@ use ruma::{
     RoomId, UInt, UserId,
 };
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     convert::{TryFrom, TryInto},
     mem,
 };
@@ -34,7 +34,7 @@ impl RoomEdus {
         event: EduEvent,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         // Remove old entry
@@ -49,7 +49,7 @@ impl RoomEdus {
                 key.rsplit(|&b| b == 0xff)
                     .next()
                     .expect("rsplit always returns an element")
-                    == user_id.to_string().as_bytes()
+                    == user_id.as_bytes()
             })
         {
             // This is the old room_latest
@@ -59,7 +59,7 @@ impl RoomEdus {
         let mut room_latest_id = prefix;
         room_latest_id.extend_from_slice(&globals.next_count()?.to_be_bytes());
         room_latest_id.push(0xff);
-        room_latest_id.extend_from_slice(&user_id.to_string().as_bytes());
+        room_latest_id.extend_from_slice(&user_id.as_bytes());
 
         self.readreceiptid_readreceipt.insert(
             room_latest_id,
@@ -76,7 +76,7 @@ impl RoomEdus {
         room_id: &RoomId,
         since: u64,
     ) -> Result<impl Iterator<Item = Result<Raw<ruma::events::AnySyncEphemeralRoomEvent>>>> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let mut first_possible_edu = prefix.clone();
@@ -102,9 +102,9 @@ impl RoomEdus {
         count: u64,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(&user_id.as_bytes());
 
         self.roomuserid_privateread
             .insert(&key, &count.to_be_bytes())?;
@@ -118,9 +118,9 @@ impl RoomEdus {
     /// Returns the private read marker.
     #[tracing::instrument(skip(self))]
     pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result<Option<u64>> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(&user_id.as_bytes());
 
         self.roomuserid_privateread.get(key)?.map_or(Ok(None), |v| {
             Ok(Some(utils::u64_from_bytes(&v).map_err(|_| {
@@ -131,9 +131,9 @@ impl RoomEdus {
 
     /// Returns the count of the last typing update in this room.
     pub fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result<u64> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(&user_id.as_bytes());
 
         Ok(self
             .roomuserid_lastprivatereadupdate
@@ -155,7 +155,7 @@ impl RoomEdus {
         timeout: u64,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let count = globals.next_count()?.to_be_bytes();
@@ -166,10 +166,10 @@ impl RoomEdus {
         room_typing_id.extend_from_slice(&count);
 
         self.typingid_userid
-            .insert(&room_typing_id, &*user_id.to_string().as_bytes())?;
+            .insert(&room_typing_id, &*user_id.as_bytes())?;
 
         self.roomid_lasttypingupdate
-            .insert(&room_id.to_string().as_bytes(), &count)?;
+            .insert(&room_id.as_bytes(), &count)?;
 
         Ok(())
     }
@@ -181,7 +181,7 @@ impl RoomEdus {
         room_id: &RoomId,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let user_id = user_id.to_string();
@@ -200,10 +200,8 @@ impl RoomEdus {
         }
 
         if found_outdated {
-            self.roomid_lasttypingupdate.insert(
+            self.roomid_lasttypingupdate
-                &room_id.to_string().as_bytes(),
+                .insert(&room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
-                &globals.next_count()?.to_be_bytes(),
-            )?;
         }
 
         Ok(())
@@ -215,7 +213,7 @@ impl RoomEdus {
         room_id: &RoomId,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let current_timestamp = utils::millis_since_unix_epoch();
@@ -248,10 +246,8 @@ impl RoomEdus {
         }
 
         if found_outdated {
-            self.roomid_lasttypingupdate.insert(
+            self.roomid_lasttypingupdate
-                &room_id.to_string().as_bytes(),
+                .insert(&room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
-                &globals.next_count()?.to_be_bytes(),
-            )?;
         }
 
         Ok(())
@@ -268,7 +264,7 @@ impl RoomEdus {
 
         Ok(self
             .roomid_lasttypingupdate
-            .get(&room_id.to_string().as_bytes())?
+            .get(&room_id.as_bytes())?
             .map_or(Ok::<_, Error>(None), |bytes| {
                 Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| {
                     Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")
@@ -281,10 +277,10 @@ impl RoomEdus {
         &self,
         room_id: &RoomId,
     ) -> Result<SyncEphemeralRoomEvent<ruma::events::typing::TypingEventContent>> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
-        let mut user_ids = Vec::new();
+        let mut user_ids = HashSet::new();
 
         for user_id in self
             .typingid_userid
@@ -299,11 +295,13 @@ impl RoomEdus {
                 )
             })
         {
-            user_ids.push(user_id?);
+            user_ids.insert(user_id?);
         }
 
         Ok(SyncEphemeralRoomEvent {
-            content: ruma::events::typing::TypingEventContent { user_ids },
+            content: ruma::events::typing::TypingEventContent {
+                user_ids: user_ids.into_iter().collect(),
+            },
         })
     }
 
@@ -322,11 +320,11 @@ impl RoomEdus {
 
         let count = globals.next_count()?.to_be_bytes();
 
-        let mut presence_id = room_id.to_string().as_bytes().to_vec();
+        let mut presence_id = room_id.as_bytes().to_vec();
         presence_id.push(0xff);
         presence_id.extend_from_slice(&count);
         presence_id.push(0xff);
-        presence_id.extend_from_slice(&presence.sender.to_string().as_bytes());
+        presence_id.extend_from_slice(&presence.sender.as_bytes());
 
         self.presenceid_presence.insert(
             presence_id,
@@ -334,7 +332,7 @@ impl RoomEdus {
         )?;
 
         self.userid_lastpresenceupdate.insert(
-            &user_id.to_string().as_bytes(),
+            &user_id.as_bytes(),
             &utils::millis_since_unix_epoch().to_be_bytes(),
         )?;
 
@@ -345,7 +343,7 @@ impl RoomEdus {
     #[tracing::instrument(skip(self))]
     pub fn ping_presence(&self, user_id: &UserId) -> Result<()> {
         self.userid_lastpresenceupdate.insert(
-            &user_id.to_string().as_bytes(),
+            &user_id.as_bytes(),
             &utils::millis_since_unix_epoch().to_be_bytes(),
         )?;
 
@@ -355,7 +353,7 @@ impl RoomEdus {
     /// Returns the timestamp of the last presence update of this user in millis since the unix epoch.
     pub fn last_presence_update(&self, user_id: &UserId) -> Result<Option<u64>> {
         self.userid_lastpresenceupdate
-            .get(&user_id.to_string().as_bytes())?
+            .get(&user_id.as_bytes())?
             .map(|bytes| {
                 utils::u64_from_bytes(&bytes).map_err(|_| {
                     Error::bad_database("Invalid timestamp in userid_lastpresenceupdate.")
@@ -386,7 +384,7 @@ impl RoomEdus {
                         .ok()?,
                 ))
             })
-            .take_while(|(_, timestamp)| current_timestamp - timestamp > 5 * 60_000)
+            .take_while(|(_, timestamp)| current_timestamp.saturating_sub(*timestamp) > 5 * 60_000)
         // 5 Minutes
         {
             // Send new presence events to set the user offline
@@ -398,7 +396,7 @@ impl RoomEdus {
                 .try_into()
                 .map_err(|_| Error::bad_database("Invalid UserId in userid_lastpresenceupdate."))?;
             for room_id in rooms.rooms_joined(&user_id).filter_map(|r| r.ok()) {
-                let mut presence_id = room_id.to_string().as_bytes().to_vec();
+                let mut presence_id = room_id.as_bytes().to_vec();
                 presence_id.push(0xff);
                 presence_id.extend_from_slice(&count);
                 presence_id.push(0xff);
@@ -424,7 +422,7 @@ impl RoomEdus {
             }
 
             self.userid_lastpresenceupdate.insert(
-                &user_id.to_string().as_bytes(),
+                &user_id.as_bytes(),
                 &utils::millis_since_unix_epoch().to_be_bytes(),
             )?;
         }
@@ -443,7 +441,7 @@ impl RoomEdus {
     ) -> Result<HashMap<UserId, PresenceEvent>> {
         self.presence_maintain(rooms, globals)?;
 
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let mut first_possible_edu = prefix.clone();

src/database/sending.rs

@@ -1,68 +1,80 @@
 use std::{
     collections::HashMap,
-    convert::TryFrom,
+    convert::{TryFrom, TryInto},
     fmt::Debug,
     sync::Arc,
     time::{Duration, Instant, SystemTime},
 };
 
-use crate::{appservice_server, server_server, utils, Error, PduEvent, Result};
+use crate::{
+    appservice_server, database::pusher, server_server, utils, Database, Error, PduEvent, Result,
+};
 use federation::transactions::send_transaction_message;
-use log::{info, warn};
+use log::{error, warn};
 use ring::digest;
 use rocket::futures::stream::{FuturesUnordered, StreamExt};
 use ruma::{
     api::{appservice, federation, OutgoingRequest},
-    ServerName,
+    events::{push_rules, EventType},
+    push, ServerName, UInt, UserId,
 };
 use sled::IVec;
 use tokio::{select, sync::Semaphore};
 
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum OutgoingKind {
+    Appservice(Box<ServerName>),
+    Push(Vec<u8>, Vec<u8>), // user and pushkey
+    Normal(Box<ServerName>),
+}
+
 #[derive(Clone)]
 pub struct Sending {
     /// The state for a given state hash.
-    pub(super) servernamepduids: sled::Tree, // ServernamePduId = (+)ServerName + PduId
+    pub(super) servernamepduids: sled::Tree, // ServernamePduId = (+ / $)SenderKey / ServerName / UserId + PduId
-    pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = (+)ServerName + PduId (pduid can be empty for reservation)
+    pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = (+ / $)ServerName / UserId + PduId
     pub(super) maximum_requests: Arc<Semaphore>,
 }
 
+enum TransactionStatus {
+    Running,
+    Failed(u32, Instant), // number of times failed, time of last failure
+    Retrying(u32),        // number of times failed
+}
+
 impl Sending {
-    pub fn start_handler(
+    pub fn start_handler(&self, db: &Database) {
-        &self,
-        globals: &super::globals::Globals,
-        rooms: &super::rooms::Rooms,
-        appservice: &super::appservice::Appservice,
-    ) {
         let servernamepduids = self.servernamepduids.clone();
         let servercurrentpdus = self.servercurrentpdus.clone();
-        let maximum_requests = self.maximum_requests.clone();
+
-        let rooms = rooms.clone();
+        let db = db.clone();
-        let globals = globals.clone();
-        let appservice = appservice.clone();
 
         tokio::spawn(async move {
             let mut futures = FuturesUnordered::new();
 
             // Retry requests we could not finish yet
-            let mut current_transactions = HashMap::<(Box<ServerName>, bool), Vec<IVec>>::new();
+            let mut subscriber = servernamepduids.watch_prefix(b"");
+            let mut current_transaction_status = HashMap::<Vec<u8>, TransactionStatus>::new();
 
-            for (key, server, pdu, is_appservice) in servercurrentpdus
+            let mut initial_transactions = HashMap::<OutgoingKind, Vec<Vec<u8>>>::new();
+            for (key, outgoing_kind, pdu) in servercurrentpdus
                 .iter()
                 .filter_map(|r| r.ok())
-                .filter_map(|(key, _)| Self::parse_servercurrentpdus(key).ok())
+                .filter_map(|(key, _)| {
+                    Self::parse_servercurrentpdus(&key)
+                        .ok()
+                        .map(|(k, p)| (key, k, p.to_vec()))
+                })
             {
-                if pdu.is_empty() {
+                let entry = initial_transactions
-                    // Remove old reservation key
+                    .entry(outgoing_kind.clone())
-                    servercurrentpdus.remove(key).unwrap();
-                    continue;
-                }
-
-                let entry = current_transactions
-                    .entry((server, is_appservice))
                     .or_insert_with(Vec::new);
 
                 if entry.len() > 30 {
-                    warn!("Dropping some current pdus because too many were queued. This should not happen.");
+                    warn!(
+                        "Dropping some current pdu: {:?} {:?} {:?}",
+                        key, outgoing_kind, pdu
+                    );
                     servercurrentpdus.remove(key).unwrap();
                     continue;
                 }
@@ -70,42 +82,55 @@ impl Sending {
                 entry.push(pdu);
             }
 
-            for ((server, is_appservice), pdus) in current_transactions {
+            for (outgoing_kind, pdus) in initial_transactions {
-                // Create new reservation
+                let mut prefix = match &outgoing_kind {
-                let mut prefix = if is_appservice {
+                    OutgoingKind::Appservice(server) => {
-                    b"+".to_vec()
+                        let mut p = b"+".to_vec();
-                } else {
+                        p.extend_from_slice(server.as_bytes());
-                    Vec::new()
+                        p
+                    }
+                    OutgoingKind::Push(user, pushkey) => {
+                        let mut p = b"$".to_vec();
+                        p.extend_from_slice(&user);
+                        p.push(0xff);
+                        p.extend_from_slice(&pushkey);
+                        p
+                    }
+                    OutgoingKind::Normal(server) => {
+                        let mut p = Vec::new();
+                        p.extend_from_slice(server.as_bytes());
+                        p
+                    }
                 };
-                prefix.extend_from_slice(server.as_bytes());
                 prefix.push(0xff);
-                servercurrentpdus.insert(prefix, &[]).unwrap();
+                current_transaction_status.insert(prefix, TransactionStatus::Running);
-
+                futures.push(Self::handle_event(outgoing_kind.clone(), pdus, &db));
-                futures.push(Self::handle_event(
-                    server,
-                    is_appservice,
-                    pdus,
-                    &globals,
-                    &rooms,
-                    &appservice,
-                    &maximum_requests,
-                ));
             }
 
-            let mut last_failed_try: HashMap<Box<ServerName>, (u32, Instant)> = HashMap::new();
-
-            let mut subscriber = servernamepduids.watch_prefix(b"");
             loop {
                 select! {
                     Some(response) = futures.next() => {
                         match response {
-                            Ok((server, is_appservice)) => {
+                            Ok(outgoing_kind) => {
-                                let mut prefix = if is_appservice {
+                                let mut prefix = match &outgoing_kind {
-                                    b"+".to_vec()
+                                    OutgoingKind::Appservice(server) => {
-                                } else {
+                                        let mut p = b"+".to_vec();
-                                    Vec::new()
+                                        p.extend_from_slice(server.as_bytes());
+                                        p
+                                    }
+                                    OutgoingKind::Push(user, pushkey) => {
+                                        let mut p = b"$".to_vec();
+                                        p.extend_from_slice(&user);
+                                        p.push(0xff);
+                                        p.extend_from_slice(&pushkey);
+                                        p
+                                    },
+                                    OutgoingKind::Normal(server) => {
+                                        let mut p = vec![];
+                                        p.extend_from_slice(server.as_bytes());
+                                        p
+                                    },
                                 };
-                                prefix.extend_from_slice(server.as_bytes());
                                 prefix.push(0xff);
 
                                 for key in servercurrentpdus
@@ -113,10 +138,7 @@ impl Sending {
                                     .keys()
                                     .filter_map(|r| r.ok())
                                 {
-                                    // Don't remove reservation yet
+                                    servercurrentpdus.remove(key).unwrap();
-                                    if prefix.len() != key.len() {
-                                        servercurrentpdus.remove(key).unwrap();
-                                    }
                                 }
 
                                 // Find events that have been added since starting the last request
@@ -125,7 +147,7 @@ impl Sending {
                                     .keys()
                                     .filter_map(|r| r.ok())
                                     .map(|k| {
-                                        k.subslice(prefix.len(), k.len() - prefix.len())
+                                        k[prefix.len()..].to_vec()
                                     })
                                     .take(30)
                                     .collect::<Vec<_>>();
@@ -138,93 +160,143 @@ impl Sending {
                                         servernamepduids.remove(&current_key).unwrap();
                                     }
 
-                                    futures.push(Self::handle_event(server, is_appservice, new_pdus, &globals, &rooms, &appservice, &maximum_requests));
+                                    futures.push(
+                                        Self::handle_event(
+                                            outgoing_kind.clone(),
+                                            new_pdus,
+                                            &db,
+                                        )
+                                    );
                                 } else {
-                                    servercurrentpdus.remove(&prefix).unwrap();
+                                    current_transaction_status.remove(&prefix);
-                                    // servercurrentpdus with the prefix should be empty now
                                 }
                             }
-                            Err((server, is_appservice, e)) => {
+                            Err((outgoing_kind, _)) => {
-                                info!("Couldn't send transaction to {}\n{}", server, e);
+                                let mut prefix = match &outgoing_kind {
-                                let mut prefix = if is_appservice {
+                                    OutgoingKind::Appservice(serv) => {
-                                    b"+".to_vec()
+                                        let mut p = b"+".to_vec();
-                                } else {
+                                        p.extend_from_slice(serv.as_bytes());
-                                    Vec::new()
+                                        p
+                                    },
+                                    OutgoingKind::Push(user, pushkey) => {
+                                        let mut p = b"$".to_vec();
+                                        p.extend_from_slice(&user);
+                                        p.push(0xff);
+                                        p.extend_from_slice(&pushkey);
+                                        p
+                                    },
+                                    OutgoingKind::Normal(serv) => {
+                                        let mut p = vec![];
+                                        p.extend_from_slice(serv.as_bytes());
+                                        p
+                                    },
                                 };
-                                prefix.extend_from_slice(server.as_bytes());
+
                                 prefix.push(0xff);
 
-                                last_failed_try.insert(server.clone(), match last_failed_try.get(&server) {
+                                current_transaction_status.entry(prefix).and_modify(|e| *e = match e {
-                                    Some(last_failed) => {
+                                    TransactionStatus::Running => TransactionStatus::Failed(1, Instant::now()),
-                                        (last_failed.0+1, Instant::now())
+                                    TransactionStatus::Retrying(n) => TransactionStatus::Failed(*n+1, Instant::now()),
+                                    TransactionStatus::Failed(_, _) => {
+                                        error!("Request that was not even running failed?!");
+                                        return
                                     },
-                                    None => {
-                                        (1, Instant::now())
-                                    }
                                 });
-                                servercurrentpdus.remove(&prefix).unwrap();
                             }
                         };
                     },
                     Some(event) = &mut subscriber => {
                         if let sled::Event::Insert { key, .. } = event {
+                        // New sled version:
+                        //for (_tree, key, value_opt) in &event {
+                        //    if value_opt.is_none() {
+                        //        continue;
+                        //    }
+
                             let servernamepduid = key.clone();
-                            let mut parts = servernamepduid.splitn(2, |&b| b == 0xff);
 
-                            let exponential_backoff = |(tries, instant): &(u32, Instant)| {
+                            let mut retry = false;
-                                // Fail if a request has failed recently (exponential backoff)
-                                let mut min_elapsed_duration = Duration::from_secs(60) * (*tries) * (*tries);
-                                if min_elapsed_duration > Duration::from_secs(60*60*24) {
-                                    min_elapsed_duration = Duration::from_secs(60*60*24);
-                                }
 
-                                instant.elapsed() < min_elapsed_duration
+                            if let Some((outgoing_kind, prefix, pdu_id)) = Self::parse_servercurrentpdus(&servernamepduid)
-                            };
-                            if let Some((server, is_appservice, pdu_id)) = utils::string_from_bytes(
-                                    parts
-                                        .next()
-                                        .expect("splitn will always return 1 or more elements"),
-                                )
-                                .map_err(|_| Error::bad_database("ServerName in servernamepduid bytes are invalid."))
-                                .map(|server_str| {
-                                    // Appservices start with a plus
-                                    if server_str.starts_with('+') {
-                                        (server_str[1..].to_owned(), true)
-                                    } else {
-                                        (server_str, false)
-                                    }
-                                })
-                                .and_then(|(server_str, is_appservice)| Box::<ServerName>::try_from(server_str)
-                                    .map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid.")).map(|s| (s, is_appservice)))
                                 .ok()
-                                .and_then(|(server, is_appservice)| parts
+                                .map(|(outgoing_kind, pdu_id)| {
-                                    .next()
+                                    let mut prefix = match &outgoing_kind {
-                                    .ok_or_else(|| Error::bad_database("Invalid servernamepduid in db."))
+                                        OutgoingKind::Appservice(serv) => {
-                                    .ok()
+                                            let mut p = b"+".to_vec();
-                                    .map(|pdu_id| (server, is_appservice, pdu_id))
+                                            p.extend_from_slice(serv.as_bytes());
-                                )
+                                            p
-                                .filter(|(server, is_appservice, _)| {
+                                    },
-                                    if last_failed_try.get(server).map_or(false, exponential_backoff) {
+                                        OutgoingKind::Push(user, pushkey) => {
-                                        return false;
+                                            let mut p = b"$".to_vec();
-                                    }
+                                            p.extend_from_slice(&user);
-
+                                            p.push(0xff);
-                                    let mut prefix = if *is_appservice {
+                                            p.extend_from_slice(&pushkey);
-                                        b"+".to_vec()
+                                            p
-                                    } else {
+                                        },
-                                        Vec::new()
+                                        OutgoingKind::Normal(serv) => {
+                                            let mut p = vec![];
+                                            p.extend_from_slice(serv.as_bytes());
+                                            p
+                                        },
                                     };
-                                    prefix.extend_from_slice(server.as_bytes());
                                     prefix.push(0xff);
 
-                                    servercurrentpdus
+                                    (outgoing_kind, prefix, pdu_id)
-                                        .compare_and_swap(prefix, Option::<&[u8]>::None, Some(&[])) // Try to reserve
+                                })
-                                        == Ok(Ok(()))
+                                .filter(|(_, prefix, _)| {
+                                    let entry = current_transaction_status.entry(prefix.clone());
+                                    let mut allow = true;
+
+                                    entry.and_modify(|e| match e {
+                                        TransactionStatus::Running | TransactionStatus::Retrying(_) => {
+                                            allow = false; // already running
+                                        },
+                                        TransactionStatus::Failed(tries, time) => {
+                                            // Fail if a request has failed recently (exponential backoff)
+                                            let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries);
+                                            if min_elapsed_duration > Duration::from_secs(60*60*24) {
+                                                min_elapsed_duration = Duration::from_secs(60*60*24);
+                                            }
+
+                                            if time.elapsed() < min_elapsed_duration {
+                                                allow = false;
+                                            } else {
+                                                retry = true;
+                                                *e = TransactionStatus::Retrying(*tries);
+                                            }
+                                        }
+                                    }).or_insert(TransactionStatus::Running);
+
+                                    allow
                                 })
                             {
-                                servercurrentpdus.insert(&key, &[]).unwrap();
+                                let mut pdus = Vec::new();
-                                servernamepduids.remove(&key).unwrap();
 
-                                futures.push(Self::handle_event(server, is_appservice, vec![pdu_id.into()], &globals, &rooms, &appservice, &maximum_requests));
+                                if retry {
+                                    // We retry the previous transaction
+                                    for pdu in servercurrentpdus
+                                        .scan_prefix(&prefix)
+                                        .filter_map(|r| r.ok())
+                                        .filter_map(|(key, _)| {
+                                            Self::parse_servercurrentpdus(&key)
+                                                .ok()
+                                                .map(|(_, p)| p.to_vec())
+                                        })
+                                    {
+                                        pdus.push(pdu);
+                                    }
+                                } else {
+                                    servercurrentpdus.insert(&key, &[]).unwrap();
+                                    servernamepduids.remove(&key).unwrap();
+                                    pdus.push(pdu_id.to_vec());
+                                }
+                                futures.push(
+                                    Self::handle_event(
+                                        outgoing_kind,
+                                        pdus,
+                                        &db,
+                                    )
+                                );
                             }
                         }
                     }
@@ -233,6 +305,17 @@ impl Sending {
         });
     }
 
+    #[tracing::instrument(skip(self))]
+    pub fn send_push_pdu(&self, pdu_id: &[u8], senderkey: IVec) -> Result<()> {
+        let mut key = b"$".to_vec();
+        key.extend_from_slice(&senderkey);
+        key.push(0xff);
+        key.extend_from_slice(pdu_id);
+        self.servernamepduids.insert(key, b"")?;
+
+        Ok(())
+    }
+
     #[tracing::instrument(skip(self))]
     pub fn send_pdu(&self, server: &ServerName, pdu_id: &[u8]) -> Result<()> {
         let mut key = server.as_bytes().to_vec();
@@ -255,153 +338,265 @@ impl Sending {
     }
 
     #[tracing::instrument]
-    fn calculate_hash(keys: &[IVec]) -> Vec<u8> {
+    fn calculate_hash(keys: &[Vec<u8>]) -> Vec<u8> {
         // We only hash the pdu's event ids, not the whole pdu
         let bytes = keys.join(&0xff);
         let hash = digest::digest(&digest::SHA256, &bytes);
         hash.as_ref().to_owned()
     }
 
-    #[tracing::instrument(skip(globals, rooms, appservice))]
+    #[tracing::instrument(skip(db))]
     async fn handle_event(
-        server: Box<ServerName>,
+        kind: OutgoingKind,
-        is_appservice: bool,
+        pdu_ids: Vec<Vec<u8>>,
-        pdu_ids: Vec<IVec>,
+        db: &Database,
-        globals: &super::globals::Globals,
+    ) -> std::result::Result<OutgoingKind, (OutgoingKind, Error)> {
-        rooms: &super::rooms::Rooms,
+        match &kind {
-        appservice: &super::appservice::Appservice,
+            OutgoingKind::Appservice(server) => {
-        maximum_requests: &Semaphore,
+                let pdu_jsons = pdu_ids
-    ) -> std::result::Result<(Box<ServerName>, bool), (Box<ServerName>, bool, Error)> {
+                    .iter()
-        if is_appservice {
+                    .map(|pdu_id| {
-            let pdu_jsons = pdu_ids
+                        Ok::<_, (Box<ServerName>, Error)>(
-                .iter()
+                            db.rooms
-                .map(|pdu_id| {
+                                .get_pdu_from_id(pdu_id)
-                    Ok::<_, (Box<ServerName>, Error)>(
+                                .map_err(|e| (server.clone(), e))?
-                        rooms
+                                .ok_or_else(|| {
-                            .get_pdu_from_id(pdu_id)
+                                    (
-                            .map_err(|e| (server.clone(), e))?
+                                        server.clone(),
-                            .ok_or_else(|| {
+                                        Error::bad_database(
-                                (
+                                            "[Appservice] Event in servernamepduids not found in db.",
-                                    server.clone(),
+                                        ),
-                                    Error::bad_database(
+                                    )
-                                        "Event in servernamepduids not found in db.",
+                                })?
-                                    ),
+                                .to_any_event(),
-                                )
-                            })?
-                            .to_any_event(),
-                    )
-                })
-                .filter_map(|r| r.ok())
-                .collect::<Vec<_>>();
-
-            let permit = maximum_requests.acquire().await;
-            let response = appservice_server::send_request(
-                &globals,
-                appservice
-                    .get_registration(server.as_str())
-                    .unwrap()
-                    .unwrap(), // TODO: handle error
-                appservice::event::push_events::v1::Request {
-                    events: &pdu_jsons,
-                    txn_id: &base64::encode_config(
-                        Self::calculate_hash(&pdu_ids),
-                        base64::URL_SAFE_NO_PAD,
-                    ),
-                },
-            )
-            .await
-            .map(|_response| (server.clone(), is_appservice))
-            .map_err(|e| (server, is_appservice, e));
-
-            drop(permit);
-
-            response
-        } else {
-            let pdu_jsons = pdu_ids
-                .iter()
-                .map(|pdu_id| {
-                    Ok::<_, (Box<ServerName>, Error)>(
-                        // TODO: check room version and remove event_id if needed
-                        serde_json::from_str(
-                            PduEvent::convert_to_outgoing_federation_event(
-                                rooms
-                                    .get_pdu_json_from_id(pdu_id)
-                                    .map_err(|e| (server.clone(), e))?
-                                    .ok_or_else(|| {
-                                        (
-                                            server.clone(),
-                                            Error::bad_database(
-                                                "Event in servernamepduids not found in db.",
-                                            ),
-                                        )
-                                    })?,
-                            )
-                            .json()
-                            .get(),
                         )
-                        .expect("Raw<..> is always valid"),
+                    })
+                    .filter_map(|r| r.ok())
+                    .collect::<Vec<_>>();
+                let permit = db.sending.maximum_requests.acquire().await;
+
+                let response = appservice_server::send_request(
+                    &db.globals,
+                    db.appservice
+                        .get_registration(server.as_str())
+                        .unwrap()
+                        .unwrap(), // TODO: handle error
+                    appservice::event::push_events::v1::Request {
+                        events: &pdu_jsons,
+                        txn_id: &base64::encode_config(
+                            Self::calculate_hash(&pdu_ids),
+                            base64::URL_SAFE_NO_PAD,
+                        ),
+                    },
+                )
+                .await
+                .map(|_response| kind.clone())
+                .map_err(|e| (kind, e));
+
+                drop(permit);
+
+                response
+            }
+            OutgoingKind::Push(user, pushkey) => {
+                let pdus = pdu_ids
+                    .iter()
+                    .map(|pdu_id| {
+                        Ok::<_, (Vec<u8>, Error)>(
+                            db.rooms
+                                .get_pdu_from_id(pdu_id)
+                                .map_err(|e| (pushkey.clone(), e))?
+                                .ok_or_else(|| {
+                                    (
+                                        pushkey.clone(),
+                                        Error::bad_database(
+                                            "[Push] Event in servernamepduids not found in db.",
+                                        ),
+                                    )
+                                })?,
+                        )
+                    })
+                    .filter_map(|r| r.ok())
+                    .collect::<Vec<_>>();
+
+                for pdu in pdus {
+                    // Redacted events are not notification targets (we don't send push for them)
+                    if pdu.unsigned.get("redacted_because").is_some() {
+                        continue;
+                    }
+
+                    let userid =
+                        UserId::try_from(utils::string_from_bytes(user).map_err(|_| {
+                            (
+                                OutgoingKind::Push(user.clone(), pushkey.clone()),
+                                Error::bad_database("Invalid push user string in db."),
+                            )
+                        })?)
+                        .map_err(|_| {
+                            (
+                                OutgoingKind::Push(user.clone(), pushkey.clone()),
+                                Error::bad_database("Invalid push user id in db."),
+                            )
+                        })?;
+
+                    let mut senderkey = user.clone();
+                    senderkey.push(0xff);
+                    senderkey.extend_from_slice(pushkey);
+
+                    let pusher = match db
+                        .pusher
+                        .get_pusher(&senderkey)
+                        .map_err(|e| (OutgoingKind::Push(user.clone(), pushkey.clone()), e))?
+                    {
+                        Some(pusher) => pusher,
+                        None => continue,
+                    };
+
+                    let rules_for_user = db
+                        .account_data
+                        .get::<push_rules::PushRulesEvent>(None, &userid, EventType::PushRules)
+                        .unwrap_or_default()
+                        .map(|ev| ev.content.global)
+                        .unwrap_or_else(|| push::Ruleset::server_default(&userid));
+
+                    let unread: UInt = db
+                        .rooms
+                        .notification_count(&userid, &pdu.room_id)
+                        .map_err(|e| (kind.clone(), e))?
+                        .try_into()
+                        .expect("notifiation count can't go that high");
+
+                    let permit = db.sending.maximum_requests.acquire().await;
+
+                    let _response = pusher::send_push_notice(
+                        &userid,
+                        unread,
+                        &pusher,
+                        rules_for_user,
+                        &pdu,
+                        db,
                     )
+                    .await
+                    .map(|_response| kind.clone())
+                    .map_err(|e| (kind.clone(), e));
+
+                    drop(permit);
+                }
+                Ok(OutgoingKind::Push(user.clone(), pushkey.clone()))
+            }
+            OutgoingKind::Normal(server) => {
+                let pdu_jsons = pdu_ids
+                    .iter()
+                    .map(|pdu_id| {
+                        Ok::<_, (OutgoingKind, Error)>(
+                            // TODO: check room version and remove event_id if needed
+                            serde_json::from_str(
+                                PduEvent::convert_to_outgoing_federation_event(
+                                    db.rooms
+                                        .get_pdu_json_from_id(pdu_id)
+                                        .map_err(|e| (OutgoingKind::Normal(server.clone()), e))?
+                                        .ok_or_else(|| {
+                                            (
+                                                OutgoingKind::Normal(server.clone()),
+                                                Error::bad_database(
+                                                    "[Normal] Event in servernamepduids not found in db.",
+                                                ),
+                                            )
+                                        })?,
+                                )
+                                .json()
+                                .get(),
+                            )
+                            .expect("Raw<..> is always valid"),
+                        )
+                    })
+                    .filter_map(|r| r.ok())
+                    .collect::<Vec<_>>();
+
+                let permit = db.sending.maximum_requests.acquire().await;
+
+                let response = server_server::send_request(
+                    &db.globals,
+                    &*server,
+                    send_transaction_message::v1::Request {
+                        origin: db.globals.server_name(),
+                        pdus: &pdu_jsons,
+                        edus: &[],
+                        origin_server_ts: SystemTime::now(),
+                        transaction_id: &base64::encode_config(
+                            Self::calculate_hash(&pdu_ids),
+                            base64::URL_SAFE_NO_PAD,
+                        ),
+                    },
+                )
+                .await
+                .map(|response| {
+                    for pdu in response.pdus {
+                        if pdu.1.is_err() {
+                            warn!("Failed to send to {}: {:?}", server, pdu);
+                        }
+                    }
+                    kind.clone()
                 })
-                .filter_map(|r| r.ok())
+                .map_err(|e| (kind, e));
-                .collect::<Vec<_>>();
 
-            let permit = maximum_requests.acquire().await;
+                drop(permit);
 
-            info!("sending pdus to {}: {:#?}", server, pdu_jsons);
+                response
-            let response = server_server::send_request(
+            }
-                &globals,
-                &*server,
-                send_transaction_message::v1::Request {
-                    origin: globals.server_name(),
-                    pdus: &pdu_jsons,
-                    edus: &[],
-                    origin_server_ts: SystemTime::now(),
-                    transaction_id: &base64::encode_config(
-                        Self::calculate_hash(&pdu_ids),
-                        base64::URL_SAFE_NO_PAD,
-                    ),
-                },
-            )
-            .await
-            .map(|response| {
-                info!("server response: {:?}", response);
-                (server.clone(), is_appservice)
-            })
-            .map_err(|e| (server, is_appservice, e));
-
-            drop(permit);
-
-            response
         }
     }
 
-    fn parse_servercurrentpdus(key: IVec) -> Result<(IVec, Box<ServerName>, IVec, bool)> {
+    fn parse_servercurrentpdus(key: &IVec) -> Result<(OutgoingKind, IVec)> {
-        let key2 = key.clone();
-        let mut parts = key2.splitn(2, |&b| b == 0xff);
-        let server = parts.next().expect("splitn always returns one element");
-        let pdu = parts
-            .next()
-            .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
-
-        let server = utils::string_from_bytes(&server).map_err(|_| {
-            Error::bad_database("Invalid server bytes in server_currenttransaction")
-        })?;
-
         // Appservices start with a plus
-        let (server, is_appservice) = if server.starts_with('+') {
+        Ok::<_, Error>(if key.starts_with(b"+") {
-            (&server[1..], true)
+            let mut parts = key[1..].splitn(2, |&b| b == 0xff);
-        } else {
-            (&*server, false)
-        };
 
-        Ok::<_, Error>((
+            let server = parts.next().expect("splitn always returns one element");
-            key,
+            let pdu = parts
-            Box::<ServerName>::try_from(server).map_err(|_| {
+                .next()
-                Error::bad_database("Invalid server string in server_currenttransaction")
+                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
-            })?,
+            let server = utils::string_from_bytes(&server).map_err(|_| {
-            IVec::from(pdu),
+                Error::bad_database("Invalid server bytes in server_currenttransaction")
-            is_appservice,
+            })?;
-        ))
+
+            (
+                OutgoingKind::Appservice(Box::<ServerName>::try_from(server).map_err(|_| {
+                    Error::bad_database("Invalid server string in server_currenttransaction")
+                })?),
+                IVec::from(pdu),
+            )
+        } else if key.starts_with(b"$") {
+            let mut parts = key[1..].splitn(3, |&b| b == 0xff);
+
+            let user = parts.next().expect("splitn always returns one element");
+            let pushkey = parts
+                .next()
+                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+            let pdu = parts
+                .next()
+                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+            (
+                OutgoingKind::Push(user.to_vec(), pushkey.to_vec()),
+                IVec::from(pdu),
+            )
+        } else {
+            let mut parts = key.splitn(2, |&b| b == 0xff);
+
+            let server = parts.next().expect("splitn always returns one element");
+            let pdu = parts
+                .next()
+                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+            let server = utils::string_from_bytes(&server).map_err(|_| {
+                Error::bad_database("Invalid server bytes in server_currenttransaction")
+            })?;
+
+            (
+                OutgoingKind::Normal(Box::<ServerName>::try_from(server).map_err(|_| {
+                    Error::bad_database("Invalid server string in server_currenttransaction")
+                })?),
+                IVec::from(pdu),
+            )
+        })
     }
 
     #[tracing::instrument(skip(self, globals))]

src/database/uiaa.rs

@@ -148,7 +148,7 @@ impl Uiaa {
         device_id: &DeviceId,
         uiaainfo: Option<&UiaaInfo>,
     ) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
@@ -170,7 +170,7 @@ impl Uiaa {
         device_id: &DeviceId,
         session: &str,
     ) -> Result<UiaaInfo> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 

src/database/users.rs

@@ -9,6 +9,7 @@ use ruma::{
     },
     encryption::DeviceKeys,
     events::{AnyToDeviceEvent, EventType},
+    identifiers::MxcUri,
     serde::Raw,
     DeviceId, DeviceKeyAlgorithm, DeviceKeyId, UInt, UserId,
 };
@@ -21,6 +22,7 @@ pub struct Users {
     pub(super) userid_avatarurl: sled::Tree,
     pub(super) userdeviceid_token: sled::Tree,
     pub(super) userdeviceid_metadata: sled::Tree, // This is also used to check if a device exists
+    pub(super) userid_devicelistversion: sled::Tree, // DevicelistVersion = u64
     pub(super) token_userdeviceid: sled::Tree,
 
     pub(super) onetimekeyid_onetimekeys: sled::Tree, // OneTimeKeyId = UserId + DeviceKeyId
@@ -150,21 +152,22 @@ impl Users {
     }
 
     /// Get a the avatar_url of a user.
-    pub fn avatar_url(&self, user_id: &UserId) -> Result<Option<String>> {
+    pub fn avatar_url(&self, user_id: &UserId) -> Result<Option<MxcUri>> {
         self.userid_avatarurl
             .get(user_id.to_string())?
-            .map_or(Ok(None), |bytes| {
+            .map(|bytes| {
-                Ok(Some(utils::string_from_bytes(&bytes).map_err(|_| {
+                let s = utils::string_from_bytes(&bytes)
-                    Error::bad_database("Avatar URL in db is invalid.")
+                    .map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?;
-                })?))
+                MxcUri::try_from(s).map_err(|_| Error::bad_database("Avatar URL in db is invalid."))
             })
+            .transpose()
     }
 
     /// Sets a new avatar_url or removes it if avatar_url is None.
-    pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option<String>) -> Result<()> {
+    pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option<MxcUri>) -> Result<()> {
         if let Some(avatar_url) = avatar_url {
             self.userid_avatarurl
-                .insert(user_id.to_string(), &*avatar_url)?;
+                .insert(user_id.to_string(), avatar_url.to_string().as_str())?;
         } else {
             self.userid_avatarurl.remove(user_id.to_string())?;
         }
@@ -183,10 +186,14 @@ impl Users {
         // This method should never be called for nonexistent users.
         assert!(self.exists(user_id)?);
 
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
+        self.userid_devicelistversion
+            .update_and_fetch(&user_id.as_bytes(), utils::increment)?
+            .expect("utils::increment will always put in a value");
+
         self.userdeviceid_metadata.insert(
             userdeviceid,
             serde_json::to_string(&Device {
@@ -206,7 +213,7 @@ impl Users {
 
     /// Removes a device from a user.
     pub fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
@@ -225,6 +232,10 @@ impl Users {
 
         // TODO: Remove onetimekeys
 
+        self.userid_devicelistversion
+            .update_and_fetch(&user_id.as_bytes(), utils::increment)?
+            .expect("utils::increment will always put in a value");
+
         self.userdeviceid_metadata.remove(&userdeviceid)?;
 
         Ok(())
@@ -232,7 +243,7 @@ impl Users {
 
     /// Returns an iterator over all device ids of this user.
     pub fn all_device_ids(&self, user_id: &UserId) -> impl Iterator<Item = Result<Box<DeviceId>>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         // All devices have metadata
         self.userdeviceid_metadata
@@ -252,7 +263,7 @@ impl Users {
 
     /// Replaces the access token of one device.
     pub fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
@@ -280,7 +291,7 @@ impl Users {
         one_time_key_value: &OneTimeKey,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(device_id.as_bytes());
 
@@ -303,10 +314,8 @@ impl Users {
                 .expect("OneTimeKey::to_string always works"),
         )?;
 
-        self.userid_lastonetimekeyupdate.insert(
+        self.userid_lastonetimekeyupdate
-            &user_id.to_string().as_bytes(),
+            .insert(&user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
-            &globals.next_count()?.to_be_bytes(),
-        )?;
 
         Ok(())
     }
@@ -314,7 +323,7 @@ impl Users {
     #[tracing::instrument(skip(self))]
     pub fn last_one_time_keys_update(&self, user_id: &UserId) -> Result<u64> {
         self.userid_lastonetimekeyupdate
-            .get(&user_id.to_string().as_bytes())?
+            .get(&user_id.as_bytes())?
             .map(|bytes| {
                 utils::u64_from_bytes(&bytes).map_err(|_| {
                     Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")
@@ -330,18 +339,16 @@ impl Users {
         key_algorithm: &DeviceKeyAlgorithm,
         globals: &super::globals::Globals,
     ) -> Result<Option<(DeviceKeyId, OneTimeKey)>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(device_id.as_bytes());
         prefix.push(0xff);
         prefix.push(b'"'); // Annoying quotation mark
-        prefix.extend_from_slice(key_algorithm.to_string().as_bytes());
+        prefix.extend_from_slice(key_algorithm.as_ref().as_bytes());
         prefix.push(b':');
 
-        self.userid_lastonetimekeyupdate.insert(
+        self.userid_lastonetimekeyupdate
-            &user_id.to_string().as_bytes(),
+            .insert(&user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
-            &globals.next_count()?.to_be_bytes(),
-        )?;
 
         self.onetimekeyid_onetimekeys
             .scan_prefix(&prefix)
@@ -371,7 +378,7 @@ impl Users {
         user_id: &UserId,
         device_id: &DeviceId,
     ) -> Result<BTreeMap<DeviceKeyAlgorithm, UInt>> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
@@ -407,7 +414,7 @@ impl Users {
         rooms: &super::rooms::Rooms,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
@@ -432,7 +439,7 @@ impl Users {
     ) -> Result<()> {
         // TODO: Check signatures
 
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         // Master key
@@ -530,9 +537,9 @@ impl Users {
         rooms: &super::rooms::Rooms,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut key = target_id.to_string().as_bytes().to_vec();
+        let mut key = target_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(key_id.to_string().as_bytes());
+        key.extend_from_slice(key_id.as_bytes());
 
         let mut cross_signing_key =
             serde_json::from_slice::<serde_json::Value>(&self.keyid_key.get(&key)?.ok_or(
@@ -615,14 +622,14 @@ impl Users {
                 continue;
             }
 
-            let mut key = room_id.to_string().as_bytes().to_vec();
+            let mut key = room_id.as_bytes().to_vec();
             key.push(0xff);
             key.extend_from_slice(&count);
 
             self.keychangeid_userid.insert(key, &*user_id.to_string())?;
         }
 
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&count);
         self.keychangeid_userid.insert(key, &*user_id.to_string())?;
@@ -635,7 +642,7 @@ impl Users {
         user_id: &UserId,
         device_id: &DeviceId,
     ) -> Result<Option<DeviceKeys>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(device_id.as_bytes());
 
@@ -722,7 +729,7 @@ impl Users {
         content: serde_json::Value,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut key = target_user_id.to_string().as_bytes().to_vec();
+        let mut key = target_user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(target_device_id.as_bytes());
         key.push(0xff);
@@ -749,7 +756,7 @@ impl Users {
     ) -> Result<Vec<Raw<AnyToDeviceEvent>>> {
         let mut events = Vec::new();
 
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(device_id.as_bytes());
         prefix.push(0xff);
@@ -771,7 +778,7 @@ impl Users {
         device_id: &DeviceId,
         until: u64,
     ) -> Result<()> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(device_id.as_bytes());
         prefix.push(0xff);
@@ -806,13 +813,17 @@ impl Users {
         device_id: &DeviceId,
         device: &Device,
     ) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
         // Only existing devices should be able to call this.
         assert!(self.userdeviceid_metadata.get(&userdeviceid)?.is_some());
 
+        self.userid_devicelistversion
+            .update_and_fetch(&user_id.as_bytes(), utils::increment)?
+            .expect("utils::increment will always put in a value");
+
         self.userdeviceid_metadata.insert(
             userdeviceid,
             serde_json::to_string(device)
@@ -829,7 +840,7 @@ impl Users {
         user_id: &UserId,
         device_id: &DeviceId,
     ) -> Result<Option<Device>> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
@@ -842,8 +853,18 @@ impl Users {
             })
     }
 
+    pub fn get_devicelist_version(&self, user_id: &UserId) -> Result<Option<u64>> {
+        self.userid_devicelistversion
+            .get(user_id.as_bytes())?
+            .map_or(Ok(None), |bytes| {
+                utils::u64_from_bytes(&bytes)
+                    .map_err(|_| Error::bad_database("Invalid devicelistversion in db."))
+                    .map(Some)
+            })
+    }
+
     pub fn all_devices_metadata(&self, user_id: &UserId) -> impl Iterator<Item = Result<Device>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
 
         self.userdeviceid_metadata

src/error.rs

@@ -1,18 +1,7 @@
-use std::{
-    collections::HashMap,
-    sync::RwLock,
-    time::{Duration, Instant},
-};
-
 use log::error;
-use ruma::{
+use ruma::api::client::{error::ErrorKind, r0::uiaa::UiaaInfo};
-    api::client::{error::ErrorKind, r0::uiaa::UiaaInfo},
-    events::room::message,
-};
 use thiserror::Error;
 
-use crate::{database::admin::AdminCommand, Database};
-
 #[cfg(feature = "conduit_bin")]
 use {
     crate::RumaResponse,
@@ -111,63 +100,3 @@ where
         .respond_to(r)
     }
 }
-
-pub struct ConduitLogger {
-    pub db: Database,
-    pub last_logs: RwLock<HashMap<String, Instant>>,
-}
-
-impl log::Log for ConduitLogger {
-    fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool {
-        true
-    }
-
-    fn log(&self, record: &log::Record<'_>) {
-        let output = format!("{} - {}", record.level(), record.args());
-
-        let match_mod_path =
-            |path: &str| path.starts_with("conduit::") || path.starts_with("state");
-
-        if self.enabled(record.metadata())
-            && (record.module_path().map_or(false, match_mod_path)
-                || record
-                    .module_path()
-                    .map_or(true, |path| !path.starts_with("rocket::")) // Rockets logs are annoying
-                    && record.metadata().level() <= log::Level::Warn)
-        {
-            let first_line = output
-                .lines()
-                .next()
-                .expect("lines always returns one item");
-
-            eprintln!("{}", output);
-
-            let mute_duration = match record.metadata().level() {
-                log::Level::Error => Duration::from_secs(60 * 5), // 5 minutes
-                log::Level::Warn => Duration::from_secs(60 * 60 * 24), // A day
-                _ => Duration::from_secs(60 * 60 * 24 * 7),       // A week
-            };
-
-            if self
-                .last_logs
-                .read()
-                .unwrap()
-                .get(first_line)
-                .map_or(false, |i| i.elapsed() < mute_duration)
-            // Don't post this log again for some time
-            {
-                return;
-            }
-
-            if let Ok(mut_last_logs) = &mut self.last_logs.try_write() {
-                mut_last_logs.insert(first_line.to_owned(), Instant::now());
-            }
-
-            self.db.admin.send(AdminCommand::SendMessage(
-                message::MessageEventContent::notice_plain(output),
-            ));
-        }
-    }
-
-    fn flush(&self) {}
-}

src/lib.rs

@@ -1,9 +1,9 @@
+#![allow(clippy::suspicious_else_formatting)]
 pub mod appservice_server;
 pub mod client_server;
 mod database;
 mod error;
 mod pdu;
-mod push_rules;
 mod ruma_wrapper;
 pub mod server_server;
 mod utils;

src/main.rs

@@ -1,4 +1,5 @@
 #![warn(rust_2018_idioms)]
+#![allow(clippy::suspicious_else_formatting)]
 
 pub mod appservice_server;
 pub mod client_server;
@@ -7,7 +8,6 @@ pub mod server_server;
 mod database;
 mod error;
 mod pdu;
-mod push_rules;
 mod ruma_wrapper;
 mod utils;
 
@@ -21,7 +21,6 @@ pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse};
 
 use rocket::{
     catch, catchers,
-    fairing::AdHoc,
     figment::{
         providers::{Env, Format, Toml},
         Figment,
@@ -31,26 +30,9 @@ use rocket::{
 use tracing::span;
 use tracing_subscriber::{prelude::*, Registry};
 
-fn setup_rocket() -> (rocket::Rocket, Config) {
+fn setup_rocket(config: Figment, data: Database) -> rocket::Rocket<rocket::Build> {
-    // Force log level off, so we can use our own logger
+    rocket::custom(config)
-    std::env::set_var("CONDUIT_LOG_LEVEL", "off");
+        .manage(data)
-
-    let config =
-        Figment::from(rocket::Config::release_default())
-            .merge(
-                Toml::file(Env::var("CONDUIT_CONFIG").expect(
-                    "The CONDUIT_CONFIG env var needs to be set. Example: /etc/conduit.toml",
-                ))
-                .nested(),
-            )
-            .merge(Env::prefixed("CONDUIT_").global());
-
-    let parsed_config = config
-        .extract::<Config>()
-        .expect("It looks like your config is invalid. Please take a look at the error");
-    let parsed_config2 = parsed_config.clone();
-
-    let rocket = rocket::custom(config)
         .mount(
             "/",
             routes![
@@ -77,7 +59,9 @@ fn setup_rocket() -> (rocket::Rocket, Config) {
                 client_server::get_filter_route,
                 client_server::create_filter_route,
                 client_server::set_global_account_data_route,
+                client_server::set_room_account_data_route,
                 client_server::get_global_account_data_route,
+                client_server::get_room_account_data_route,
                 client_server::set_displayname_route,
                 client_server::get_displayname_route,
                 client_server::set_avatar_url_route,
@@ -155,6 +139,7 @@ fn setup_rocket() -> (rocket::Rocket, Config) {
                 client_server::get_key_changes_route,
                 client_server::get_pushers_route,
                 client_server::set_pushers_route,
+                // client_server::third_party_route,
                 client_server::upgrade_room_route,
                 server_server::get_server_version_route,
                 server_server::get_server_keys_route,
@@ -162,34 +147,53 @@ fn setup_rocket() -> (rocket::Rocket, Config) {
                 server_server::get_public_rooms_route,
                 server_server::get_public_rooms_filtered_route,
                 server_server::send_transaction_message_route,
+                server_server::get_event_route,
                 server_server::get_missing_events_route,
+                server_server::get_room_state_ids_route,
+                server_server::create_join_event_template_route,
+                server_server::create_join_event_route,
+                server_server::create_invite_route,
+                server_server::get_devices_route,
+                server_server::get_room_information_route,
                 server_server::get_profile_information_route,
             ],
         )
-        .register(catchers![
+        .register(
-            not_found_catcher,
+            "/",
-            forbidden_catcher,
+            catchers![
-            unknown_token_catcher,
+                not_found_catcher,
-            missing_token_catcher,
+                forbidden_catcher,
-            bad_json_catcher
+                unknown_token_catcher,
-        ])
+                missing_token_catcher,
-        .attach(AdHoc::on_attach("Config", |rocket| async {
+                bad_json_catcher
-            let data = Database::load_or_create(parsed_config2)
+            ],
-                .await
+        )
-                .expect("config is valid");
-
-            data.sending
-                .start_handler(&data.globals, &data.rooms, &data.appservice);
-
-            Ok(rocket.manage(data))
-        }));
-
-    (rocket, parsed_config)
 }
 
 #[rocket::main]
 async fn main() {
-    let (rocket, config) = setup_rocket();
+    // Force log level off, so we can use our own logger
+    std::env::set_var("CONDUIT_LOG_LEVEL", "off");
+
+    let raw_config =
+        Figment::from(rocket::Config::release_default())
+            .merge(
+                Toml::file(Env::var("CONDUIT_CONFIG").expect(
+                    "The CONDUIT_CONFIG env var needs to be set. Example: /etc/conduit.toml",
+                ))
+                .nested(),
+            )
+            .merge(Env::prefixed("CONDUIT_").global());
+
+    let config = raw_config
+        .extract::<Config>()
+        .expect("It looks like your config is invalid. Please take a look at the error");
+
+    let db = Database::load_or_create(config.clone())
+        .await
+        .expect("config is valid");
+
+    db.sending.start_handler(&db);
 
     if config.allow_jaeger {
         let (tracer, _uninstall) = opentelemetry_jaeger::new_pipeline()
@@ -201,16 +205,13 @@ async fn main() {
 
         let root = span!(tracing::Level::INFO, "app_start", work_units = 2);
         let _enter = root.enter();
-
-        rocket.launch().await.unwrap();
     } else {
-        pretty_env_logger::init();
+        std::env::set_var("CONDUIT_LOG", config.log);
-
+        pretty_env_logger::init_custom_env("CONDUIT_LOG");
-        let root = span!(tracing::Level::INFO, "app_start", work_units = 2);
-        let _enter = root.enter();
-
-        rocket.launch().await.unwrap();
     }
+
+    let rocket = setup_rocket(raw_config, db);
+    rocket.launch().await.unwrap();
 }
 
 #[catch(404)]

src/pdu.rs

@@ -1,10 +1,11 @@
 use crate::Error;
+use log::error;
 use ruma::{
     events::{
         pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent,
         AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent,
     },
-    serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw},
+    serde::{CanonicalJsonObject, CanonicalJsonValue, Raw},
     EventId, RoomId, RoomVersionId, ServerName, ServerSigningKeyId, UInt, UserId,
 };
 use serde::{Deserialize, Serialize};
@@ -165,22 +166,17 @@ impl PduEvent {
 
     #[tracing::instrument(skip(self))]
     pub fn to_sync_state_event(&self) -> Raw<AnySyncStateEvent> {
-        let json = format!(
+        let json = json!({
-            r#"{{"content":{},"type":"{}","event_id":"{}","sender":"{}","origin_server_ts":{},"unsigned":{},"state_key":"{}"}}"#,
+            "content": self.content,
-            self.content,
+            "type": self.kind,
-            self.kind,
+            "event_id": self.event_id,
-            self.event_id,
+            "sender": self.sender,
-            self.sender,
+            "origin_server_ts": self.origin_server_ts,
-            self.origin_server_ts,
+            "unsigned": self.unsigned,
-            serde_json::to_string(&self.unsigned).expect("Map::to_string always works"),
+            "state_key": self.state_key,
-            self.state_key
+        });
-                .as_ref()
-                .expect("state events have state keys")
-        );
 
-        Raw::from_json(
+        serde_json::from_value(json).expect("Raw::from_value always works")
-            serde_json::value::RawValue::from_string(json).expect("our string is valid json"),
-        )
     }
 
     #[tracing::instrument(skip(self))]
@@ -242,7 +238,7 @@ impl PduEvent {
     ) -> Result<Self, serde_json::Error> {
         json.insert(
             "event_id".to_string(),
-            to_canonical_value(event_id).expect("event_id is a valid Value"),
+            CanonicalJsonValue::String(event_id.as_str().to_owned()),
         );
 
         serde_json::from_value(serde_json::to_value(json).expect("valid JSON"))
@@ -322,8 +318,11 @@ impl Ord for PduEvent {
 /// Returns a tuple of the new `EventId` and the PDU as a `BTreeMap<String, CanonicalJsonValue>`.
 pub(crate) fn gen_event_id_canonical_json(
     pdu: &Raw<ruma::events::pdu::Pdu>,
-) -> (EventId, CanonicalJsonObject) {
+) -> crate::Result<(EventId, CanonicalJsonObject)> {
-    let value = serde_json::from_str(pdu.json().get()).expect("A Raw<...> is always valid JSON");
+    let value = serde_json::from_str(pdu.json().get()).map_err(|e| {
+        error!("{:?}: {:?}", pdu, e);
+        Error::BadServerResponse("Invalid PDU in server response")
+    })?;
 
     let event_id = EventId::try_from(&*format!(
         "${}",
@@ -332,7 +331,7 @@ pub(crate) fn gen_event_id_canonical_json(
     ))
     .expect("ruma's reference hashes are valid event ids");
 
-    (event_id, value)
+    Ok((event_id, value))
 }
 
 /// Build the start of a PDU in order to add it to the `Database`.

src/push_rules.rs

@@ -1,256 +0,0 @@
-use ruma::{
-    push::{
-        Action, ConditionalPushRule, ConditionalPushRuleInit, ContentPushRule, OverridePushRule,
-        PatternedPushRule, PatternedPushRuleInit, PushCondition, RoomMemberCountIs, Ruleset, Tweak,
-        UnderridePushRule,
-    },
-    UserId,
-};
-
-pub fn default_pushrules(user_id: &UserId) -> Ruleset {
-    let mut rules = Ruleset::default();
-
-    rules.add(ContentPushRule(contains_user_name_rule(&user_id)));
-
-    for rule in vec![
-        master_rule(),
-        suppress_notices_rule(),
-        invite_for_me_rule(),
-        member_event_rule(),
-        contains_display_name_rule(),
-        tombstone_rule(),
-        roomnotif_rule(),
-    ] {
-        rules.add(OverridePushRule(rule));
-    }
-
-    for rule in vec![
-        call_rule(),
-        encrypted_room_one_to_one_rule(),
-        room_one_to_one_rule(),
-        message_rule(),
-        encrypted_rule(),
-    ] {
-        rules.add(UnderridePushRule(rule));
-    }
-
-    rules
-}
-
-pub fn master_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::DontNotify],
-        default: true,
-        enabled: false,
-        rule_id: ".m.rule.master".to_owned(),
-        conditions: vec![],
-    }
-    .into()
-}
-
-pub fn suppress_notices_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::DontNotify],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.suppress_notices".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "content.msgtype".to_owned(),
-            pattern: "m.notice".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn invite_for_me_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(false)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.invite_for_me".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "content.membership".to_owned(),
-            pattern: "m.invite".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn member_event_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::DontNotify],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.member_event".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "content.membership".to_owned(),
-            pattern: "type".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn contains_display_name_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(true)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.contains_display_name".to_owned(),
-        conditions: vec![PushCondition::ContainsDisplayName],
-    }
-    .into()
-}
-
-pub fn tombstone_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::Notify, Action::SetTweak(Tweak::Highlight(true))],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.tombstone".to_owned(),
-        conditions: vec![
-            PushCondition::EventMatch {
-                key: "type".to_owned(),
-                pattern: "m.room.tombstone".to_owned(),
-            },
-            PushCondition::EventMatch {
-                key: "state_key".to_owned(),
-                pattern: "".to_owned(),
-            },
-        ],
-    }
-    .into()
-}
-
-pub fn roomnotif_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::Notify, Action::SetTweak(Tweak::Highlight(true))],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.roomnotif".to_owned(),
-        conditions: vec![
-            PushCondition::EventMatch {
-                key: "content.body".to_owned(),
-                pattern: "@room".to_owned(),
-            },
-            PushCondition::SenderNotificationPermission {
-                key: "room".to_owned(),
-            },
-        ],
-    }
-    .into()
-}
-
-pub fn contains_user_name_rule(user_id: &UserId) -> PatternedPushRule {
-    PatternedPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(true)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.contains_user_name".to_owned(),
-        pattern: user_id.localpart().to_owned(),
-    }
-    .into()
-}
-
-pub fn call_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("ring".to_owned())),
-            Action::SetTweak(Tweak::Highlight(false)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.call".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "type".to_owned(),
-            pattern: "m.call.invite".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn encrypted_room_one_to_one_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(false)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.encrypted_room_one_to_one".to_owned(),
-        conditions: vec![
-            PushCondition::RoomMemberCount {
-                is: RoomMemberCountIs::from(2_u32.into()..),
-            },
-            PushCondition::EventMatch {
-                key: "type".to_owned(),
-                pattern: "m.room.encrypted".to_owned(),
-            },
-        ],
-    }
-    .into()
-}
-
-pub fn room_one_to_one_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(false)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.room_one_to_one".to_owned(),
-        conditions: vec![
-            PushCondition::RoomMemberCount {
-                is: RoomMemberCountIs::from(2_u32.into()..),
-            },
-            PushCondition::EventMatch {
-                key: "type".to_owned(),
-                pattern: "m.room.message".to_owned(),
-            },
-        ],
-    }
-    .into()
-}
-
-pub fn message_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::Notify, Action::SetTweak(Tweak::Highlight(false))],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.message".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "type".to_owned(),
-            pattern: "m.room.message".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn encrypted_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::Notify, Action::SetTweak(Tweak::Highlight(false))],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.encrypted".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "type".to_owned(),
-            pattern: "m.room.encrypted".to_owned(),
-        }],
-    }
-    .into()
-}

src/ruma_wrapper.rs

@@ -1,173 +1,298 @@
 use crate::Error;
 use ruma::{
-    api::{AuthScheme, IncomingRequest, OutgoingRequest},
+    api::OutgoingResponse,
     identifiers::{DeviceId, UserId},
     Outgoing,
 };
-use std::{
+use std::ops::Deref;
-    convert::{TryFrom, TryInto},
-    ops::Deref,
-};
 
 #[cfg(feature = "conduit_bin")]
 use {
-    crate::utils,
+    crate::{server_server, utils},
     log::{debug, warn},
     rocket::{
-        data::{
+        data::{self, ByteUnit, Data, FromData},
-            ByteUnit, Data, FromDataFuture, FromTransformedData, Transform, TransformFuture,
-            Transformed,
-        },
         http::Status,
         outcome::Outcome::*,
         response::{self, Responder},
         tokio::io::AsyncReadExt,
         Request, State,
     },
+    ruma::{
+        api::{AuthScheme, IncomingRequest},
+        signatures::CanonicalJsonValue,
+        ServerName,
+    },
+    std::collections::BTreeMap,
+    std::convert::TryFrom,
     std::io::Cursor,
 };
 
 /// This struct converts rocket requests into ruma structs by converting them into http requests
 /// first.
-pub struct Ruma<T: Outgoing + OutgoingRequest> {
+pub struct Ruma<T: Outgoing> {
     pub body: T::Incoming,
     pub sender_user: Option<UserId>,
     pub sender_device: Option<Box<DeviceId>>,
-    pub json_body: Option<Box<serde_json::value::RawValue>>, // This is None when body is not a valid string
+    // This is None when body is not a valid string
+    pub json_body: Option<Box<serde_json::value::RawValue>>,
     pub from_appservice: bool,
 }
 
 #[cfg(feature = "conduit_bin")]
-impl<'a, T: Outgoing + OutgoingRequest> FromTransformedData<'a> for Ruma<T>
+#[rocket::async_trait]
+impl<'a, T: Outgoing> FromData<'a> for Ruma<T>
 where
     T::Incoming: IncomingRequest,
 {
     type Error = ();
-    type Owned = Data;
-    type Borrowed = Self::Owned;
 
-    fn transform<'r>(
+    async fn from_data(request: &'a Request<'_>, data: Data) -> data::Outcome<Self, Self::Error> {
-        _req: &'r Request<'_>,
+        let metadata = T::Incoming::METADATA;
-        data: Data,
+        let db = request
-    ) -> TransformFuture<'r, Self::Owned, Self::Error> {
+            .guard::<State<'_, crate::Database>>()
-        Box::pin(async move { Transform::Owned(Success(data)) })
+            .await
-    }
+            .expect("database was loaded");
 
-    fn from_data(
+        // Get token from header or query value
-        request: &'a Request<'_>,
+        let token = request
-        outcome: Transformed<'a, Self>,
+            .headers()
-    ) -> FromDataFuture<'a, Self, Self::Error> {
+            .get_one("Authorization")
-        Box::pin(async move {
+            .map(|s| s[7..].to_owned()) // Split off "Bearer "
-            let data = rocket::try_outcome!(outcome.owned());
+            .or_else(|| request.query_value("access_token").and_then(|r| r.ok()));
-            let db = request
-                .guard::<State<'_, crate::Database>>()
-                .await
-                .expect("database was loaded");
 
-            // Get token from header or query value
+        let limit = db.globals.max_request_size();
-            let token = request
+        let mut handle = data.open(ByteUnit::Byte(limit.into()));
-                .headers()
+        let mut body = Vec::new();
-                .get_one("Authorization")
+        handle.read_to_end(&mut body).await.unwrap();
-                .map(|s| s[7..].to_owned()) // Split off "Bearer "
-                .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok()));
 
-            let (sender_user, sender_device, from_appservice) = if let Some((_id, registration)) =
+        let (sender_user, sender_device, from_appservice) = if let Some((_id, registration)) = db
-                db.appservice
+            .appservice
-                    .iter_all()
+            .iter_all()
-                    .filter_map(|r| r.ok())
+            .filter_map(|r| r.ok())
-                    .find(|(_id, registration)| {
+            .find(|(_id, registration)| {
-                        registration
+                registration
-                            .get("as_token")
+                    .get("as_token")
-                            .and_then(|as_token| as_token.as_str())
+                    .and_then(|as_token| as_token.as_str())
-                            .map_or(false, |as_token| token.as_deref() == Some(as_token))
+                    .map_or(false, |as_token| token.as_deref() == Some(as_token))
-                    }) {
+            }) {
-                match T::METADATA.authentication {
+            match metadata.authentication {
-                    AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
+                AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
-                        let user_id = request.get_query_value::<String>("user_id").map_or_else(
+                    let user_id = request.query_value::<String>("user_id").map_or_else(
-                            || {
+                        || {
-                                UserId::parse_with_server_name(
+                            UserId::parse_with_server_name(
-                                    registration
+                                registration
-                                        .get("sender_localpart")
+                                    .get("sender_localpart")
-                                        .unwrap()
-                                        .as_str()
-                                        .unwrap(),
-                                    db.globals.server_name(),
-                                )
-                                .unwrap()
-                            },
-                            |string| {
-                                UserId::try_from(string.expect("parsing to string always works"))
                                     .unwrap()
-                            },
+                                    .as_str()
-                        );
+                                    .unwrap(),
+                                db.globals.server_name(),
+                            )
+                            .unwrap()
+                        },
+                        |string| {
+                            UserId::try_from(string.expect("parsing to string always works"))
+                                .unwrap()
+                        },
+                    );
+
+                    if !db.users.exists(&user_id).unwrap() {
+                        // Forbidden
+                        return Failure((Status::raw(580), ()));
+                    }
+
+                    // TODO: Check if appservice is allowed to be that user
+                    (Some(user_id), None, true)
+                }
+                AuthScheme::ServerSignatures => (None, None, true),
+                AuthScheme::None => (None, None, true),
+            }
+        } else {
+            match metadata.authentication {
+                AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
+                    if let Some(token) = token {
+                        match db.users.find_from_token(&token).unwrap() {
+                            // Unknown Token
+                            None => return Failure((Status::raw(581), ())),
+                            Some((user_id, device_id)) => {
+                                (Some(user_id), Some(device_id.into()), false)
+                            }
+                        }
+                    } else {
+                        // Missing Token
+                        return Failure((Status::raw(582), ()));
+                    }
+                }
+                AuthScheme::ServerSignatures => {
+                    // Get origin from header
+                    let x_matrix = match request.headers().get_one("Authorization").map(|s| {
+                        // Split off "X-Matrix " and parse the rest
+                        s[9..]
+                            .split_terminator(',')
+                            .map(|field| {
+                                let mut splits = field.splitn(2, '=');
+                                (splits.next(), splits.next().map(|s| s.trim_matches('"')))
+                            })
+                            .collect::<BTreeMap<_, _>>()
+                    }) {
+                        Some(t) => t,
+                        None => {
+                            warn!("No Authorization header");
 
-                        if !db.users.exists(&user_id).unwrap() {
                             // Forbidden
                             return Failure((Status::raw(580), ()));
                         }
+                    };
 
-                        // TODO: Check if appservice is allowed to be that user
+                    let origin_str = match x_matrix.get(&Some("origin")) {
-                        (Some(user_id), None, true)
+                        Some(Some(o)) => *o,
-                    }
+                        _ => {
-                    AuthScheme::ServerSignatures => (None, None, true),
+                            warn!("Invalid X-Matrix header origin field: {:?}", x_matrix);
-                    AuthScheme::None => (None, None, true),
+
-                }
+                            // Forbidden
-            } else {
+                            return Failure((Status::raw(580), ()));
-                match T::METADATA.authentication {
+                        }
-                    AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
+                    };
-                        if let Some(token) = token {
+
-                            match db.users.find_from_token(&token).unwrap() {
+                    let origin = match Box::<ServerName>::try_from(origin_str) {
-                                // Unknown Token
+                        Ok(s) => s,
-                                None => return Failure((Status::raw(581), ())),
+                        _ => {
-                                Some((user_id, device_id)) => {
+                            warn!(
-                                    (Some(user_id), Some(device_id.into()), false)
+                                "Invalid server name in X-Matrix header origin field: {:?}",
-                                }
+                                x_matrix
+                            );
+
+                            // Forbidden
+                            return Failure((Status::raw(580), ()));
+                        }
+                    };
+
+                    let key = match x_matrix.get(&Some("key")) {
+                        Some(Some(k)) => *k,
+                        _ => {
+                            warn!("Invalid X-Matrix header key field: {:?}", x_matrix);
+
+                            // Forbidden
+                            return Failure((Status::raw(580), ()));
+                        }
+                    };
+
+                    let sig = match x_matrix.get(&Some("sig")) {
+                        Some(Some(s)) => *s,
+                        _ => {
+                            warn!("Invalid X-Matrix header sig field: {:?}", x_matrix);
+
+                            // Forbidden
+                            return Failure((Status::raw(580), ()));
+                        }
+                    };
+
+                    let json_body = serde_json::from_slice::<CanonicalJsonValue>(&body);
+
+                    let mut request_map = BTreeMap::<String, CanonicalJsonValue>::new();
+
+                    if let Ok(json_body) = json_body {
+                        request_map.insert("content".to_owned(), json_body);
+                    };
+
+                    request_map.insert(
+                        "method".to_owned(),
+                        CanonicalJsonValue::String(request.method().to_string()),
+                    );
+                    request_map.insert(
+                        "uri".to_owned(),
+                        CanonicalJsonValue::String(request.uri().to_string()),
+                    );
+                    request_map.insert(
+                        "origin".to_owned(),
+                        CanonicalJsonValue::String(origin.as_str().to_owned()),
+                    );
+                    request_map.insert(
+                        "destination".to_owned(),
+                        CanonicalJsonValue::String(db.globals.server_name().as_str().to_owned()),
+                    );
+
+                    let mut origin_signatures = BTreeMap::new();
+                    origin_signatures
+                        .insert(key.to_owned(), CanonicalJsonValue::String(sig.to_owned()));
+
+                    let mut signatures = BTreeMap::new();
+                    signatures.insert(
+                        origin.as_str().to_owned(),
+                        CanonicalJsonValue::Object(origin_signatures),
+                    );
+
+                    request_map.insert(
+                        "signatures".to_owned(),
+                        CanonicalJsonValue::Object(signatures),
+                    );
+
+                    let keys = match server_server::fetch_signing_keys(
+                        &db,
+                        &origin,
+                        vec![&key.to_owned()],
+                    )
+                    .await
+                    {
+                        Ok(b) => b,
+                        Err(e) => {
+                            warn!("Failed to fetch signing keys: {}", e);
+
+                            // Forbidden
+                            return Failure((Status::raw(580), ()));
+                        }
+                    };
+
+                    let mut pub_key_map = BTreeMap::new();
+                    pub_key_map.insert(origin.as_str().to_owned(), keys);
+
+                    match ruma::signatures::verify_json(&pub_key_map, &request_map) {
+                        Ok(()) => (None, None, false),
+                        Err(e) => {
+                            warn!("Failed to verify json request from {}: {}", origin, e);
+
+                            if request.uri().to_string().contains('@') {
+                                warn!("Request uri contained '@' character. Make sure your reverse proxy gives Conduit the raw uri (apache: use nocanon)");
                             }
-                        } else {
+
-                            // Missing Token
+                            // Forbidden
-                            return Failure((Status::raw(582), ()));
+                            return Failure((Status::raw(580), ()));
                         }
                     }
-                    AuthScheme::ServerSignatures => (None, None, false),
-                    AuthScheme::None => (None, None, false),
                 }
-            };
+                AuthScheme::None => (None, None, false),
-
-            let mut http_request = http::Request::builder()
-                .uri(request.uri().to_string())
-                .method(&*request.method().to_string());
-            for header in request.headers().iter() {
-                http_request = http_request.header(header.name.as_str(), &*header.value);
             }
+        };
 
-            let limit = db.globals.max_request_size();
+        let mut http_request = http::Request::builder()
-            let mut handle = data.open(ByteUnit::Byte(limit.into()));
+            .uri(request.uri().to_string())
-            let mut body = Vec::new();
+            .method(&*request.method().to_string());
-            handle.read_to_end(&mut body).await.unwrap();
+        for header in request.headers().iter() {
+            http_request = http_request.header(header.name.as_str(), &*header.value);
+        }
 
-            let http_request = http_request.body(body.clone()).unwrap();
+        let http_request = http_request.body(&*body).unwrap();
-            debug!("{:?}", http_request);
+        debug!("{:?}", http_request);
-            match <T::Incoming as IncomingRequest>::try_from_http_request(http_request) {
+        match <T::Incoming as IncomingRequest>::try_from_http_request(http_request) {
-                Ok(t) => Success(Ruma {
+            Ok(t) => Success(Ruma {
-                    body: t,
+                body: t,
-                    sender_user,
+                sender_user,
-                    sender_device,
+                sender_device,
-                    // TODO: Can we avoid parsing it again? (We only need this for append_pdu)
+                // TODO: Can we avoid parsing it again? (We only need this for append_pdu)
-                    json_body: utils::string_from_bytes(&body)
+                json_body: utils::string_from_bytes(&body)
-                        .ok()
+                    .ok()
-                        .and_then(|s| serde_json::value::RawValue::from_string(s).ok()),
+                    .and_then(|s| serde_json::value::RawValue::from_string(s).ok()),
-                    from_appservice,
+                from_appservice,
-                }),
+            }),
-                Err(e) => {
+            Err(e) => {
-                    warn!("{:?}", e);
+                warn!("{:?}", e);
-                    Failure((Status::raw(583), ()))
+                Failure((Status::raw(583), ()))
-                }
             }
-        })
+        }
     }
 }
 
-impl<T: Outgoing + OutgoingRequest> Deref for Ruma<T> {
+impl<T: Outgoing> Deref for Ruma<T> {
     type Target = T::Incoming;
 
     fn deref(&self) -> &Self::Target {
@@ -178,9 +303,9 @@ impl<T: Outgoing + OutgoingRequest> Deref for Ruma<T> {
 /// This struct converts ruma responses into rocket http responses.
 pub type ConduitResult<T> = std::result::Result<RumaResponse<T>, Error>;
 
-pub struct RumaResponse<T: TryInto<http::Response<Vec<u8>>>>(pub T);
+pub struct RumaResponse<T: OutgoingResponse>(pub T);
 
-impl<T: TryInto<http::Response<Vec<u8>>>> From<T> for RumaResponse<T> {
+impl<T: OutgoingResponse> From<T> for RumaResponse<T> {
     fn from(t: T) -> Self {
         Self(t)
     }
@@ -189,40 +314,38 @@ impl<T: TryInto<http::Response<Vec<u8>>>> From<T> for RumaResponse<T> {
 #[cfg(feature = "conduit_bin")]
 impl<'r, 'o, T> Responder<'r, 'o> for RumaResponse<T>
 where
-    T: Send + TryInto<http::Response<Vec<u8>>>,
+    T: Send + OutgoingResponse,
-    T::Error: Send,
     'o: 'r,
 {
     fn respond_to(self, _: &'r Request<'_>) -> response::Result<'o> {
-        let http_response: Result<http::Response<_>, _> = self.0.try_into();
+        let http_response = self
-        match http_response {
+            .0
-            Ok(http_response) => {
+            .try_into_http_response::<Vec<u8>>()
-                let mut response = rocket::response::Response::build();
+            .map_err(|_| Status::InternalServerError)?;
 
-                let status = http_response.status();
+        let mut response = rocket::response::Response::build();
-                response.raw_status(status.into(), "");
 
-                for header in http_response.headers() {
+        let status = http_response.status();
-                    response
+        response.raw_status(status.into(), "");
-                        .raw_header(header.0.to_string(), header.1.to_str().unwrap().to_owned());
-                }
 
-                let http_body = http_response.into_body();
+        for header in http_response.headers() {
-
+            response.raw_header(header.0.to_string(), header.1.to_str().unwrap().to_owned());
-                response.sized_body(http_body.len(), Cursor::new(http_body));
-
-                response.raw_header("Access-Control-Allow-Origin", "*");
-                response.raw_header(
-                    "Access-Control-Allow-Methods",
-                    "GET, POST, PUT, DELETE, OPTIONS",
-                );
-                response.raw_header(
-                    "Access-Control-Allow-Headers",
-                    "Origin, X-Requested-With, Content-Type, Accept, Authorization",
-                );
-                response.ok()
-            }
-            Err(_) => Err(Status::InternalServerError),
         }
+
+        let http_body = http_response.into_body();
+
+        response.sized_body(http_body.len(), Cursor::new(http_body));
+
+        response.raw_header("Access-Control-Allow-Origin", "*");
+        response.raw_header(
+            "Access-Control-Allow-Methods",
+            "GET, POST, PUT, DELETE, OPTIONS",
+        );
+        response.raw_header(
+            "Access-Control-Allow-Headers",
+            "Origin, X-Requested-With, Content-Type, Accept, Authorization",
+        );
+        response.raw_header("Access-Control-Max-Age", "86400");
+        response.ok()
     }
 }

src/utils.rs

@@ -2,7 +2,6 @@ use argon2::{Config, Variant};
 use cmp::Ordering;
 use rand::prelude::*;
 use ruma::serde::{try_from_json_map, CanonicalJsonError, CanonicalJsonObject};
-use sled::IVec;
 use std::{
     cmp,
     convert::TryInto,
@@ -71,9 +70,9 @@ pub fn calculate_hash(password: &str) -> Result<String, argon2::Error> {
 }
 
 pub fn common_elements(
-    mut iterators: impl Iterator<Item = impl Iterator<Item = IVec>>,
+    mut iterators: impl Iterator<Item = impl Iterator<Item = Vec<u8>>>,
-    check_order: impl Fn(&IVec, &IVec) -> Ordering,
+    check_order: impl Fn(&[u8], &[u8]) -> Ordering,
-) -> Option<impl Iterator<Item = IVec>> {
+) -> Option<impl Iterator<Item = Vec<u8>>> {
     let first_iterator = iterators.next()?;
     let mut other_iterators = iterators.map(|i| i.peekable()).collect::<Vec<_>>();
 

tests/Complement.Dockerfile

@@ -1,3 +1,4 @@
+# For use in our CI only. This requires a build artifact created by a previous run pipline stage to be placed in cached_target/release/conduit
 FROM valkum/docker-rust-ci:latest as builder
 WORKDIR /workdir
 
@@ -9,7 +10,9 @@ ARG SCCACHE_ENDPOINT
 ARG SCCACHE_S3_USE_SSL
 
 COPY . .
-RUN test -e cached_target/release/conduit || cargo build --release
+RUN mkdir -p target/release
+RUN test -e cached_target/release/conduit && cp cached_target/release/conduit target/release/conduit || cargo build --release
+
 
 FROM valkum/docker-rust-ci:latest
 WORKDIR /workdir
@@ -17,24 +20,30 @@ WORKDIR /workdir
 RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.2.1/caddy_2.2.1_linux_amd64.tar.gz"
 RUN tar xzf caddy_2.2.1_linux_amd64.tar.gz
 
-COPY --from=builder /workdir/target/debug/conduit /workdir/conduit
+COPY cached_target/release/conduit /workdir/conduit
+RUN chmod +x /workdir/conduit
+RUN chmod +x /workdir/caddy
 
-COPY Rocket-example.toml Rocket.toml
+COPY conduit-example.toml conduit.toml
 
 ENV SERVER_NAME=localhost
 ENV ROCKET_LOG=normal
+ENV CONDUIT_CONFIG=/workdir/conduit.toml
 
-RUN sed -i "s/port = 14004/port = 8008/g" Rocket.toml
+RUN sed -i "s/port = 6167/port = 8008/g" conduit.toml
-RUN echo "federation_enabled = true" >> Rocket.toml
+RUN echo "allow_federation = true" >> conduit.toml
+RUN echo "allow_encryption = true" >> conduit.toml
+RUN echo "allow_registration = true" >> conduit.toml
+RUN echo "log = \"info,rocket=info,_=off,sled=off\"" >> conduit.toml
+RUN sed -i "s/address = \"127.0.0.1\"/address = \"0.0.0.0\"/g" conduit.toml
 
 # Enabled Caddy auto cert generation for complement provided CA.
-RUN echo '{"apps":{"http":{"https_port":8448,"servers":{"srv0":{"listen":[":8448"],"routes":[{"match":[{"host":["your.server.name"]}],"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"localhost:8008"}]}]}]}],"terminal":true}],"tls_connection_policies": [{"match": {"sni": ["your.server.name"]}}]}}},"pki": {"certificate_authorities": {"local": {"name": "Complement CA","root": {"certificate": "/ca/ca.crt","private_key": "/ca/ca.key"},"intermediate": {"certificate": "/ca/ca.crt","private_key": "/ca/ca.key"}}}},"tls":{"automation":{"policies":[{"subjects":["your.server.name"],"issuer":{"module":"internal"},"on_demand":true},{"issuer":{"module":"internal", "ca": "local"}}]}}}}' > caddy.json 
+RUN echo '{"logging":{"logs":{"default":{"level":"WARN"}}}, "apps":{"http":{"https_port":8448,"servers":{"srv0":{"listen":[":8448"],"routes":[{"match":[{"host":["your.server.name"]}],"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"127.0.0.1:8008"}]}]}]}],"terminal":true}],"tls_connection_policies": [{"match": {"sni": ["your.server.name"]}}]}}},"pki": {"certificate_authorities": {"local": {"name": "Complement CA","root": {"certificate": "/ca/ca.crt","private_key": "/ca/ca.key"},"intermediate": {"certificate": "/ca/ca.crt","private_key": "/ca/ca.key"}}}},"tls":{"automation":{"policies":[{"subjects":["your.server.name"],"issuer":{"module":"internal"},"on_demand":true},{"issuer":{"module":"internal", "ca": "local"}}]}}}}' > caddy.json 
  
 EXPOSE 8008 8448
 
 CMD ([ -z "${COMPLEMENT_CA}" ] && echo "Error: Need Complement PKI support" && true) || \
-    sed -i "s/server_name = \"your.server.name\"/server_name = \"${SERVER_NAME}\"/g" Rocket.toml && \
+    sed -i "s/#server_name = \"your.server.name\"/server_name = \"${SERVER_NAME}\"/g" conduit.toml && \
     sed -i "s/your.server.name/${SERVER_NAME}/g" caddy.json && \
     /workdir/caddy start --config caddy.json > /dev/null && \
     /workdir/conduit
-    

tests/sytest/sytest-whitelist

@@ -1,71 +1,335 @@
+/event/ does not allow access to events before the user joined
+/event/ on joined room works
+/event/ on non world readable room does not work
 /joined_members return joined members
 /joined_rooms returns only joined rooms
+/whois
 3pid invite join valid signature but revoked keys are rejected
 3pid invite join valid signature but unreachable ID server are rejected
 3pid invite join with wrong but valid signature are rejected
+A change to displayname should appear in incremental /sync
+A full_state incremental update returns all state
+A full_state incremental update returns only recent timeline
+A message sent after an initial sync appears in the timeline of an incremental sync.
+A next_batch token can be used in the v1 messages API
+A pair of events which redact each other should be ignored
+A pair of servers can establish a join in a v2 room
+A prev_batch token can be used in the v1 messages API
+AS can create a user
+AS can create a user with an underscore
+AS can create a user with inhibit_login
+AS can set avatar for ghosted users
+AS can set displayname for ghosted users
+AS can't set displayname for random users
 AS cannot create users outside its own namespace
+AS user (not ghost) can join room without registering
+AS user (not ghost) can join room without registering, with user_id query param
+After changing password, a different session no longer works by default
+After changing password, can log in with new password
+After changing password, can't log in with old password
+After changing password, different sessions can optionally be kept
+After changing password, existing session still works
 After deactivating account, can't log in with an email
+After deactivating account, can't log in with password
 Alias creators can delete alias with no ops
 Alias creators can delete canonical alias with no ops
 Alternative server names do not cause a routing loop
+An event which redacts an event in a different room should be ignored
+An event which redacts itself should be ignored
+Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list
+Backfill checks the events requested belong to the room
+Backfill works correctly with history visibility set to joined
+Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination
+Banned servers cannot /event_auth
+Banned servers cannot /invite
+Banned servers cannot /make_join
+Banned servers cannot /make_leave
+Banned servers cannot /send_join
+Banned servers cannot /send_leave
+Banned servers cannot backfill
+Banned servers cannot get missing events
+Banned servers cannot get room state
+Banned servers cannot get room state ids
+Banned servers cannot send events
+Banned user is kicked and may not rejoin until unbanned
 Both GET and PUT work
+Can /sync newly created room
 Can add account data
+Can add account data to room
 Can add tag
+Can claim one time key using POST
+Can claim remote one time key using POST
 Can create filter
+Can deactivate account
 Can delete canonical alias
+Can download file 'ascii'
+Can download file 'name with spaces'
+Can download file 'name;with;semicolons'
+Can download filter
+Can download specifying a different ASCII file name
+Can download specifying a different Unicode file name
+Can download with Unicode file name locally
+Can download with Unicode file name over federation
+Can download without a file name locally
+Can download without a file name over federation
+Can forget room you've been kicked from
+Can get 'm.room.name' state for a departed room (SPEC-216)
+Can get account data without syncing
+Can get remote public room list
+Can get room account data without syncing
+Can get rooms/{roomId}/members
+Can get rooms/{roomId}/members for a departed room (SPEC-216)
+Can get rooms/{roomId}/state for a departed room (SPEC-216)
 Can invite users to invite-only rooms
 Can list tags for a room
 Can logout all devices
 Can logout current device
+Can paginate public room list
+Can pass a JSON filter as a query parameter
+Can query device keys using POST
+Can query remote device keys using POST
+Can query specific device keys using POST
 Can re-join room if re-invited
 Can read configuration endpoint
+Can receive redactions from regular users over federation in room version 1
+Can receive redactions from regular users over federation in room version 2
+Can receive redactions from regular users over federation in room version 3
+Can receive redactions from regular users over federation in room version 4
+Can receive redactions from regular users over federation in room version 5
+Can receive redactions from regular users over federation in room version 6
 Can recv a device message using /sync
+Can recv a device message using /sync 
+Can recv device messages over federation 
 Can recv device messages until they are acknowledged
+Can recv device messages until they are acknowledged 
+Can reject invites over federation for rooms with version 1
+Can reject invites over federation for rooms with version 2
+Can reject invites over federation for rooms with version 3
+Can reject invites over federation for rooms with version 4
+Can reject invites over federation for rooms with version 5
+Can reject invites over federation for rooms with version 6
 Can remove tag
+Can search public room list
 Can send a message directly to a device using PUT /sendToDevice
+Can send a message directly to a device using PUT /sendToDevice 
 Can send a to-device message to two users which both receive it using /sync
+Can send image in room message
 Can send messages with a wildcard device id
+Can send messages with a wildcard device id 
 Can send messages with a wildcard device id to two devices
+Can send messages with a wildcard device id to two devices 
 Can sync
+Can sync a joined room
+Can sync a room with a message with a transaction id
+Can sync a room with a single message
+Can upload device keys
 Can upload with ASCII file name
 Can upload with Unicode file name
 Can upload without a file name
+Can't deactivate account with wrong password
+Can't forget room you're still in
+Changes to state are included in an gapped incremental sync
+Changes to state are included in an incremental sync
 Changing the actions of an unknown default rule fails with 404
 Changing the actions of an unknown rule fails with 404
 Checking local federation server
+Creators can delete alias
 Current state appears in timeline in private history
 Current state appears in timeline in private history with many messages before
+DELETE /device/{deviceId}
+DELETE /device/{deviceId} requires UI auth user to match device owner
+DELETE /device/{deviceId} with no body gives a 401
 Deleted tags appear in an incremental v2 /sync
 Deleting a non-existent alias should return a 404
+Device list doesn't change if remote server is down
+Device messages over federation wake up /sync 
 Device messages wake up /sync
+Device messages wake up /sync 
 Device messages with the same txn_id are deduplicated
+Device messages with the same txn_id are deduplicated 
+Enabling an unknown default rule fails with 404
+Event size limits
+Event with an invalid signature in the send_join response should not cause room join to fail
 Events come down the correct room
+Events whose auth_events are in the wrong room do not mess up the room state
+Existing members see new members' join events
+Federation key API allows unsigned requests for keys
+Federation key API can act as a notary server via a GET request
+Federation key API can act as a notary server via a POST request
+Federation rejects inbound events where the prev_events cannot be found
+Fetching eventstream a second time doesn't yield the message again
+Forgetting room does not show up in v2 /sync
+Full state sync includes joined rooms
+GET /capabilities is present and well formed for registered user 
 GET /device/{deviceId}
 GET /device/{deviceId} gives a 404 for unknown devices
 GET /devices
+GET /directory/room/:room_alias yields room ID
+GET /events initially
 GET /events with negative 'limit'
 GET /events with non-numeric 'limit'
 GET /events with non-numeric 'timeout'
+GET /initialSync initially
 GET /joined_rooms lists newly-created room
 GET /login yields a set of flows
 GET /media/r0/download can fetch the value again
 GET /profile/:user_id/avatar_url publicly accessible
 GET /profile/:user_id/displayname publicly accessible
+GET /publicRooms includes avatar URLs
 GET /publicRooms lists newly-created room
+GET /publicRooms lists rooms
+GET /r0/capabilities is not public
 GET /register yields a set of flows
+GET /rooms/:room_id/joined_members fetches my membership
+GET /rooms/:room_id/messages returns a message
 GET /rooms/:room_id/state fetches entire room state
 GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership
+GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event
+GET /rooms/:room_id/state/m.room.name gets name
+GET /rooms/:room_id/state/m.room.power_levels can fetch levels
+GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels
+GET /rooms/:room_id/state/m.room.topic gets topic
+Get left notifs for other users in sync and /keys/changes when user leaves
+Getting messages going forward is limited for a departed room (SPEC-216)
 Getting push rules doesn't corrupt the cache SYN-390
+Getting state IDs checks the events requested belong to the room
+Getting state checks the events requested belong to the room
+Ghost user must register before joining room
+Guest non-joined user cannot call /events on default room
+Guest non-joined user cannot call /events on invited room
+Guest non-joined user cannot call /events on joined room
+Guest non-joined user cannot call /events on shared room
+Guest non-joined users can get individual state for world_readable rooms
+Guest non-joined users can get individual state for world_readable rooms after leaving
+Guest non-joined users can get state for world_readable rooms
+Guest non-joined users cannot room initalSync for non-world_readable rooms
+Guest non-joined users cannot send messages to guest_access rooms if not joined
+Guest user can set display names
+Guest user cannot call /events globally
+Guest user cannot upgrade other users
+Guest users can accept invites to private rooms over federation
+Guest users can join guest_access rooms
+Guest users can send messages to guest_access rooms if joined
+If a device list update goes missing, the server resyncs on the next one
+If remote user leaves room we no longer receive device updates
+If remote user leaves room, changes device and rejoins we see update in /keys/changes
+If remote user leaves room, changes device and rejoins we see update in sync
+Inbound /make_join rejects attempts to join rooms where all users have left
+Inbound /v1/make_join rejects remote attempts to join local users to rooms
+Inbound /v1/send_join rejects incorrectly-signed joins
+Inbound /v1/send_join rejects joins from other servers
+Inbound /v1/send_leave rejects leaves from other servers
+Inbound federation accepts a second soft-failed event
+Inbound federation accepts attempts to join v2 rooms from servers with support
+Inbound federation can backfill events
+Inbound federation can get public room list
+Inbound federation can get state for a room
+Inbound federation can get state_ids for a room
+Inbound federation can query profile data
+Inbound federation can query room alias directory
+Inbound federation can receive events
+Inbound federation can receive invites via v1 API
+Inbound federation can receive invites via v2 API
+Inbound federation can receive redacted events
+Inbound federation can receive v1 /send_join
+Inbound federation can receive v2 /send_join
+Inbound federation can return events
+Inbound federation can return missing events for invite visibility
+Inbound federation can return missing events for world_readable visibility
+Inbound federation correctly soft fails events
+Inbound federation of state requires event_id as a mandatory paramater
+Inbound federation of state_ids requires event_id as a mandatory paramater
+Inbound federation rejects attempts to join v1 rooms from servers without v1 support
+Inbound federation rejects attempts to join v2 rooms from servers lacking version support
+Inbound federation rejects attempts to join v2 rooms from servers only supporting v1
+Inbound federation rejects invite rejections which include invalid JSON for room version 6
+Inbound federation rejects invites which include invalid JSON for room version 6
+Inbound federation rejects receipts from wrong remote
+Inbound federation rejects remote attempts to join local users to rooms
+Inbound federation rejects remote attempts to kick local users to rooms
+Inbound federation rejects typing notifications from wrong remote
+Inbound: send_join rejects invalid JSON for room version 6
+Invalid JSON floats
+Invalid JSON integers
+Invalid JSON special values
+Invited user can reject invite
+Invited user can reject invite over federation
+Invited user can reject invite over federation for empty room
+Invited user can reject invite over federation several times
+Invited user can see room metadata
+Inviting an AS-hosted user asks the AS server
+Lazy loading parameters in the filter are strictly boolean
+Left rooms appear in the leave section of full state sync
+Local delete device changes appear in v2 /sync
+Local device key changes appear in /keys/changes
+Local device key changes appear in v2 /sync
+Local device key changes get to remote servers
+Local new device changes appear in v2 /sync
+Local non-members don't see posted message events
+Local room members can get room messages
+Local room members see posted message events
+Local update device changes appear in v2 /sync
+Local users can peek by room alias
+Local users can peek into world_readable rooms by room ID
+Message history can be paginated
+Message history can be paginated over federation
+Name/topic keys are correct
+New account data appears in incremental v2 /sync
+New read receipts appear in incremental v2 /sync
+New room members see their own join event
+New users appear in /keys/changes
+Newly banned rooms appear in the leave section of incremental sync
+Newly joined room is included in an incremental sync
+Newly joined room is included in an incremental sync after invite
+Newly left rooms appear in the leave section of gapped sync
+Newly left rooms appear in the leave section of incremental sync
 Newly updated tags appear in an incremental v2 /sync
+Non-numeric ports in server names are rejected
+Outbound federation can backfill events
+Outbound federation can query profile data
+Outbound federation can query room alias directory
+Outbound federation can query v1 /send_join
+Outbound federation can query v2 /send_join
+Outbound federation can request missing events
+Outbound federation can send events
+Outbound federation can send invites via v1 API
+Outbound federation can send invites via v2 API
+Outbound federation can send room-join requests
+Outbound federation correctly handles unsupported room versions
+Outbound federation passes make_join failures through to the client
+Outbound federation rejects backfill containing invalid JSON for events in room version 6
+Outbound federation rejects m.room.create events with an unknown room version
+Outbound federation rejects send_join responses with no m.room.create event
+Outbound federation sends receipts
+Outbound federation will ignore a missing event with bad JSON for room version 6
+POST /createRoom creates a room with the given version
+POST /createRoom ignores attempts to set the room version via creation_content
 POST /createRoom makes a private room
 POST /createRoom makes a private room with invites
+POST /createRoom makes a public room
+POST /createRoom makes a room with a name
+POST /createRoom makes a room with a topic
+POST /createRoom rejects attempts to create rooms with numeric versions
+POST /createRoom rejects attempts to create rooms with unknown versions
+POST /createRoom with creation content
+POST /join/:room_alias can join a room
+POST /join/:room_alias can join a room with custom content
 POST /join/:room_id can join a room
+POST /join/:room_id can join a room with custom content
 POST /login as non-existing user is rejected
 POST /login can log in as a user
 POST /login can log in as a user with just the local part of the id
 POST /login returns the same device_id as that in the request
 POST /login wrong password is rejected
 POST /media/r0/upload can create an upload
+POST /redact disallows redaction of event in different room
+POST /register allows registration of usernames with '-'
+POST /register allows registration of usernames with '.'
+POST /register allows registration of usernames with '/'
+POST /register allows registration of usernames with '3'
+POST /register allows registration of usernames with '='
+POST /register allows registration of usernames with '_'
+POST /register allows registration of usernames with 'q'
 POST /register can create a user
 POST /register downcases capitals in usernames
 POST /register rejects registration of usernames with '!'
@@ -88,41 +352,161 @@ POST /rooms/:room_id/ban can ban a user
 POST /rooms/:room_id/invite can send an invite
 POST /rooms/:room_id/join can join a room
 POST /rooms/:room_id/leave can leave a room
+POST /rooms/:room_id/read_markers can create read marker
+POST /rooms/:room_id/receipt can create receipts
+POST /rooms/:room_id/redact/:event_id as original message sender redacts message
+POST /rooms/:room_id/redact/:event_id as power user redacts message
+POST /rooms/:room_id/redact/:event_id as random user does not redact message
+POST /rooms/:room_id/send/:event_type sends a message
 POST /rooms/:room_id/state/m.room.name sets name
 POST /rooms/:room_id/state/m.room.topic sets topic
 POST /rooms/:room_id/upgrade can upgrade a room version
+POST rejects invalid utf-8 in JSON
 POSTed media can be thumbnailed
 PUT /device/{deviceId} gives a 404 for unknown devices
 PUT /device/{deviceId} updates device fields
 PUT /directory/room/:room_alias creates alias
 PUT /profile/:user_id/avatar_url sets my avatar
 PUT /profile/:user_id/displayname sets my name
+PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id
+PUT /rooms/:room_id/send/:event_type/:txn_id sends a message
 PUT /rooms/:room_id/state/m.room.power_levels can set levels
+PUT /rooms/:room_id/typing/:user_id sets typing notification
 PUT power_levels should not explode if the old power levels were empty
+Peeked rooms only turn up in the sync for the device who peeked them
+Previously left rooms don't appear in the leave section of sync
 Push rules come down in an initial /sync
 Read markers appear in incremental v2 /sync
 Read markers appear in initial v2 /sync
 Read markers can be updated
+Read receipts appear in initial v2 /sync
+Real non-joined user cannot call /events on default room
+Real non-joined user cannot call /events on invited room
+Real non-joined user cannot call /events on joined room
+Real non-joined user cannot call /events on shared room
+Real non-joined users can get individual state for world_readable rooms
+Real non-joined users can get individual state for world_readable rooms after leaving
+Real non-joined users can get state for world_readable rooms
+Real non-joined users cannot room initalSync for non-world_readable rooms
+Real non-joined users cannot send messages to guest_access rooms if not joined
+Receipts must be m.read
+Redaction of a redaction redacts the redaction reason
 Regular users can add and delete aliases in the default room configuration
+Regular users can add and delete aliases when m.room.aliases is restricted
+Regular users cannot create room aliases within the AS namespace
+Regular users cannot register within the AS namespace
+Remote media can be thumbnailed
+Remote room alias queries can handle Unicode
+Remote room members also see posted message events
+Remote room members can get room messages
+Remote user can backfill in a room with version 1
+Remote user can backfill in a room with version 2
+Remote user can backfill in a room with version 3
+Remote user can backfill in a room with version 4
+Remote user can backfill in a room with version 5
+Remote user can backfill in a room with version 6
+Remote users can join room by alias
+Remote users may not join unfederated rooms
+Request to logout with invalid an access token is rejected
+Request to logout without an access token is rejected
+Room aliases can contain Unicode
 Room creation reports m.room.create to myself
 Room creation reports m.room.member to myself
+Room members can join a room with an overridden displayname
+Room members can override their displayname on a room-specific basis
+Room state at a rejected message event is the same as its predecessor
+Room state at a rejected state event is the same as its predecessor
 Rooms a user is invited to appear in an incremental sync
 Rooms a user is invited to appear in an initial sync
+Rooms can be created with an initial invite list (SYN-205)
+Server correctly handles incoming m.device_list_update
+Server correctly handles transactions that break edu limits
+Server correctly resyncs when client query keys and there is no remote cache
+Server correctly resyncs when server leaves and rejoins a room
+Server rejects invalid JSON in a version 6 room
 Setting room topic reports m.room.topic to myself
+Should not be able to take over the room by pretending there is no PL event
 Should reject keys claiming to belong to a different user
+State from remote users is included in the state in the initial sync
+State from remote users is included in the timeline in an incremental sync
+State is included in the timeline in the initial sync
+Sync can be polled for updates
+Sync is woken up for leaves
+Syncing a new room with a large timeline limit isn't limited
 Tags appear in an initial v2 /sync
 Trying to get push rules with unknown rule_id fails with 404
+Typing can be explicitly stopped
 Typing events appear in gapped sync
 Typing events appear in incremental sync
 Typing events appear in initial sync
+Typing notification sent to local room members
+Typing notifications also sent to remote room members
+Typing notifications don't leak
 Uninvited users cannot join the room
+Unprivileged users can set m.room.topic if it only needs level 0
 User appears in user directory
+User can create and send/receive messages in a room with version 1
+User can create and send/receive messages in a room with version 2
+User can create and send/receive messages in a room with version 3
+User can create and send/receive messages in a room with version 4
+User can create and send/receive messages in a room with version 5
+User can create and send/receive messages in a room with version 6
+User can invite local user to room with version 1
+User can invite local user to room with version 2
+User can invite local user to room with version 3
+User can invite local user to room with version 4
+User can invite local user to room with version 5
+User can invite local user to room with version 6
+User can invite remote user to room with version 1
+User can invite remote user to room with version 2
+User can invite remote user to room with version 3
+User can invite remote user to room with version 4
+User can invite remote user to room with version 5
+User can invite remote user to room with version 6
 User directory correctly update on display name change
 User in dir while user still shares private rooms
 User in shared private room does appear in user directory
 User is offline if they set_presence=offline in their sync
+User signups are forbidden from starting with '_'
+Users can't delete other's aliases
+Users cannot invite a user that is already in the room
+Users cannot invite themselves to a room
+Users cannot kick users from a room they are not in
+Users cannot kick users who have already left a room
+Users cannot set ban powerlevel higher than their own
+Users cannot set kick powerlevel higher than their own
+Users cannot set notifications powerlevel higher than their own
+Users cannot set redact powerlevel higher than their own
+Users receive device_list updates for their own devices
 Users with sufficient power-level can delete other's aliases
 Version responds 200 OK with valid structure
+We can't peek into rooms with invited history_visibility
+We can't peek into rooms with joined history_visibility
+We can't peek into rooms with shared history_visibility
+We don't send redundant membership state across incremental syncs by default
 We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462)
+We should see our own leave event, even if history_visibility is restricted (SYN-662)
+Wildcard device messages over federation wake up /sync 
 Wildcard device messages wake up /sync
+Wildcard device messages wake up /sync 
+avatar_url updates affect room member events
+displayname updates affect room member events
+local user can join room with version 1
+local user can join room with version 2
+local user can join room with version 3
+local user can join room with version 4
+local user can join room with version 5
+local user can join room with version 6
+m.room.history_visibility == "joined" allows/forbids appropriately for Guest users
+m.room.history_visibility == "joined" allows/forbids appropriately for Real users
+m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users
+m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users
 query for user with no keys returns empty key dict
+remote user can join room with version 1
+remote user can join room with version 2
+remote user can join room with version 3
+remote user can join room with version 4
+remote user can join room with version 5
+remote user can join room with version 6
+setting 'm.room.name' respects room powerlevel
+setting 'm.room.power_levels' respects room powerlevel