fix: long prev event fetch times for huge rooms

Bump version of alpine docker image to 3.14

See merge request famedly/conduit!150

.github/ISSUE_TEMPLATE/Issue.md

@@ -0,0 +1,11 @@
+---
+name: "Issue with / Feature Request for Conduit"
+about: "Please file issues on GitLab: https://gitlab.com/famedly/conduit/-/issues/new"
+title: "CLOSE ME"
+---
+
+
+
+**⚠️ Conduit development does not happen on GitHub. Issues opened here will not be addressed** 
+
+Please open issues on GitLab: https://gitlab.com/famedly/conduit/-/issues/new

.gitignore

@@ -1,4 +1,65 @@
-/target
-**/*.rs.bk
+# CMake
+cmake-build-*/
 
+# IntelliJ
+.idea/
+out/
+*.iml
+modules.xml
+*.ipr
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# Linux backup files
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+# Rust
+/target/
+
+### vscode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows shortcuts
+*.lnk
+
+# Conduit
 Rocket.toml
+conduit.toml
+conduit.db
+
+# Etc.
+**/*.rs.bk

.gitlab-ci.yml

@@ -0,0 +1,301 @@
+stages:
+  - build
+  - build docker image
+  - test
+  - upload artifacts
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+  FF_USE_FASTZIP: 1
+  CACHE_COMPRESSION_LEVEL: fastest
+
+test:cargo:
+  stage: "test"
+  needs: []
+  image: "rust:latest"
+  tags: ["docker"]
+  variables:
+    CARGO_HOME: "cargohome"
+  cache:
+    paths:
+      - target
+      - cargohome
+    key: test_cache
+  interruptible: true
+  before_script:
+    - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
+    - apt-get update -yqq
+    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
+    - rustup component add clippy rustfmt
+  script:
+    - rustc --version && cargo --version  # Print version info for debugging
+    - cargo fmt --all -- --check
+    - cargo test --workspace --verbose --locked
+    - cargo clippy
+
+test:sytest:
+  stage: "test"
+  allow_failure: true
+  needs:
+    - "build:cargo:x86_64-unknown-linux-musl"
+  image:
+    name: "valkum/sytest-conduit:latest"
+    entrypoint: [""]
+  tags: ["docker"]
+  variables:
+    PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
+  before_script:
+    - "mkdir -p /app"
+    - "cp ./conduit-x86_64-unknown-linux-musl /app/conduit"
+    - "chmod +x /app/conduit"
+    - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
+    - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
+    - "cd /"
+  script:
+    - "SYTEST_EXIT_CODE=0"
+    - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
+    - "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap"
+    - "exit $SYTEST_EXIT_CODE"
+  artifacts:
+    when: always
+    paths:
+      - "$CI_PROJECT_DIR/sytest.xml"
+      - "$CI_PROJECT_DIR/results.tap"
+    reports:
+      junit: "$CI_PROJECT_DIR/sytest.xml"
+
+
+test:register:element-web-stable:
+  stage: "test"
+  needs:
+    - "build:cargo:x86_64-unknown-linux-gnu"
+  image: "buildkite/puppeteer:latest"
+  tags: ["docker"]
+  interruptible: true
+  script:
+    - "CONDUIT_CONFIG=tests/test-config.toml ./conduit-x86_64-unknown-linux-gnu > conduit.log &"
+    - "cd tests/client-element-web/"
+    - "npm install puppeteer"
+    - "node test-element-web-registration.js \"https://app.element.io/\" \"http://localhost:6167\""
+    - "killall --regexp \"conduit\""
+    - "cd ../.."
+    - "cat conduit.log"
+  artifacts:
+    paths:
+      - "tests/client-element-web/*.png"
+      - "*.log"
+    expire_in: 1 week
+    when: always
+  retry: 1
+
+
+# --------------------------------------------------------------------- #
+#  Cargo: Compiling for different architectures                         #
+# --------------------------------------------------------------------- #
+
+.build-cargo-shared-settings:
+  stage: "build"
+  needs: []
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+  interruptible: true
+  image: "rust:latest"
+  tags: ["docker"]
+  cache:
+    paths:
+      - cargohome
+      - target/
+    key: "build_cache-$TARGET"
+  before_script:
+    - 'echo "Building for target $TARGET"'
+    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
+    - 'apt-get update -yqq'
+    - 'echo "Installing packages: $NEEDED_PACKAGES"'
+    - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
+    - "rustup target add $TARGET"
+  script:
+    # Set some cargo tuning here, because targets overwrite the 'variables'
+    - "export CARGO_INCREMENTAL=true"
+    - "export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16"
+    - "export CARGO_PROFILE_RELEASE_LTO=thin"
+    - time cargo build --target $TARGET --release
+    - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
+
+build:cargo:x86_64-unknown-linux-gnu:
+  extends: .build-cargo-shared-settings
+  variables:
+    TARGET: "x86_64-unknown-linux-gnu"
+  rules:
+    - if: "$CI_COMMIT_BRANCH"
+  artifacts:
+    name: "conduit-x86_64-unknown-linux-gnu"
+    paths:
+      - "conduit-x86_64-unknown-linux-gnu"
+    expose_as: "Release binary x86_64-unknown-linux-gnu"
+
+build:cargo:armv7-unknown-linux-gnueabihf:
+  extends: .build-cargo-shared-settings
+  variables:
+    TARGET: "armv7-unknown-linux-gnueabihf"
+    NEEDED_PACKAGES: "build-essential gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc6-dev-armhf-cross"
+    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc
+    CC_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-gcc
+    CXX_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-g++
+  artifacts:
+    name: "conduit-armv7-unknown-linux-gnueabihf"
+    paths:
+      - "conduit-armv7-unknown-linux-gnueabihf"
+    expose_as: "Release binary armv7-unknown-linux-gnueabihf"
+
+build:cargo:aarch64-unknown-linux-gnu:
+  extends: .build-cargo-shared-settings
+  variables:
+    TARGET: "aarch64-unknown-linux-gnu"
+    NEEDED_PACKAGES: "build-essential gcc-8-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross"
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
+    CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
+    CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
+    TARGET_CC: "/usr/bin/aarch64-linux-gnu-gcc-8"
+    TARGET_AR: "/usr/bin/aarch64-linux-gnu-gcc-ar-8"
+  artifacts:
+    name: "conduit-aarch64-unknown-linux-gnu"
+    paths:
+      - "conduit-aarch64-unknown-linux-gnu"
+    expose_as: "Release binary aarch64-unknown-linux-gnu"
+
+build:cargo:x86_64-unknown-linux-musl:
+  extends: .build-cargo-shared-settings
+  image: "rust:alpine"
+  rules:
+    - if: '$CI_COMMIT_BRANCH' # Always run
+  variables:
+    TARGET: "x86_64-unknown-linux-musl"
+  before_script:
+    - 'echo "Building for target $TARGET"'
+    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
+    - "rustup target add $TARGET"
+    - "apk add libc-dev"
+  artifacts:
+    name: "conduit-x86_64-unknown-linux-musl"
+    paths:
+      - "conduit-x86_64-unknown-linux-musl"
+    expose_as: "Release binary x86_64-unknown-linux-musl"
+
+
+# --------------------------------------------------------------------- #
+#  Cargo: Compiling deb packages for different architectures            #
+# --------------------------------------------------------------------- #
+
+
+.build-cargo-deb-shared-settings:
+  stage: "build"
+  needs: []
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+  interruptible: true
+  image: "rust:latest"
+  tags: ["docker"]
+  cache:
+    paths:
+      - cargohome
+      - target/
+    key: "build_cache-deb-$TARGET"
+  before_script:
+    - 'echo "Building debian package for target $TARGET"'
+    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
+    - 'apt-get update -yqq'
+    - 'echo "Installing packages: $NEEDED_PACKAGES"'
+    - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
+    - "rustup target add $TARGET"
+    - "cargo install cargo-deb"
+  script:
+    - time cargo deb --target $TARGET
+    - 'mv target/$TARGET/debian/*.deb "conduit-$TARGET.deb"'
+
+build:cargo-deb:x86_64-unknown-linux-gnu:
+  extends: .build-cargo-deb-shared-settings
+  variables:
+    TARGET: "x86_64-unknown-linux-gnu"
+    NEEDED_PACKAGES: ""
+  artifacts:
+    name: "conduit-x86_64-unknown-linux-gnu.deb"
+    paths:
+      - "conduit-x86_64-unknown-linux-gnu.deb"
+    expose_as: "Debian Package x86_64"
+
+
+
+# --------------------------------------------------------------------- #
+#  Create and publish docker image                                      #
+# --------------------------------------------------------------------- #
+
+.docker-shared-settings:
+  stage: "build docker image"
+  needs: []
+  interruptible: true
+  image:
+    name: "gcr.io/kaniko-project/executor:debug"
+    entrypoint: [""]
+  tags: ["docker"]
+  variables:
+    # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache
+    KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache"
+  before_script:
+    - "mkdir -p /kaniko/.docker"
+    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'
+
+
+# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
+build:docker:main:
+  extends: .docker-shared-settings
+  needs:
+    - "build:cargo:x86_64-unknown-linux-musl"
+  script:
+    - >
+      /kaniko/executor
+      $KANIKO_CACHE_ARGS
+      --context $CI_PROJECT_DIR
+      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
+      --build-arg "GIT_REF=$CI_COMMIT_REF_NAME"
+      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
+      --destination "$CI_REGISTRY_IMAGE/conduit:latest"
+      --destination "$CI_REGISTRY_IMAGE/conduit:alpine"
+      --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+
+
+# --------------------------------------------------------------------- #
+#  Store binaries as package so they have download urls                 #
+# --------------------------------------------------------------------- #
+
+publish:package:
+  stage: "upload artifacts"
+  needs:
+    - "build:cargo:x86_64-unknown-linux-gnu"
+    - "build:cargo:armv7-unknown-linux-gnueabihf"
+    - "build:cargo:aarch64-unknown-linux-gnu"
+    - "build:cargo:x86_64-unknown-linux-musl"
+    - "build:cargo-deb:x86_64-unknown-linux-gnu"
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+  image: curlimages/curl:latest
+  tags: ["docker"]
+  variables:
+    GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
+  script:
+    - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu "${BASE_URL}/conduit-x86_64-unknown-linux-gnu"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-gnueabihf "${BASE_URL}/conduit-armv7-unknown-linux-gnueabihf"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-gnu "${BASE_URL}/conduit-aarch64-unknown-linux-gnu"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu.deb "${BASE_URL}/conduit-x86_64-unknown-linux-gnu.deb"'
+
+

.gitlab/issue_templates/Bug Report.md

@@ -0,0 +1,19 @@
+<!--
+If you're requesting a new feature, that isn't part of this project yet,
+then please consider filling out a "Feature Request" instead!
+
+If you need a hand setting up your conduit server, feel free to ask for help in the
+Conduit Matrix Chat: https://matrix.to/#/#conduit:fachschaften.org.
+-->
+
+### Description
+<!-- What did you do and what happened? Why is that bad? -->
+
+### System Configuration
+<!-- Other data that might help us debug this issue, like os, conduit version, database backend -->
+
+Conduit Version:
+Database backend (default is sqlite): sqlite
+
+
+/label ~conduit

.gitlab/issue_templates/Feature Request.md

@@ -0,0 +1,17 @@
+<!--
+If you want to report a bug or an error,
+then please consider filling out a "Bug Report" instead!
+-->
+
+
+### Is your feature request related to a problem? Please describe.
+
+<!-- Eg. I'm always frustrated when [...] -->
+
+
+### Describe the solution you'd like
+
+
+
+
+/label ~conduit

.gitlab/merge_request_templates/MR.md

@@ -0,0 +1,8 @@
+
+<!-- Please describe your changes here -->
+
+-----------------------------------------------------------------------------
+
+- [ ] I ran `cargo fmt` and `cargo test`
+- [ ] I agree to release my code and all other changes of this MR under the Apache-2.0 license
+

.vscode/settings.json

@@ -0,0 +1,3 @@
+{
+    "rust-analyzer.procMacro.enable": true
+}

APPSERVICES.md

@@ -0,0 +1,48 @@
+# Setting up Appservices
+
+## Getting help
+
+If you run into any problems while setting up an Appservice, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
+
+## Tested appservices
+
+Here are some appservices we tested and that work with Conduit:
+- matrix-appservice-discord
+- mautrix-hangouts
+- mautrix-telegram
+
+## Set up the appservice
+
+Follow whatever instructions are given by the appservice. This usually includes
+downloading, changing its config (setting domain, homeserver url, port etc.)
+and later starting it.
+
+At some point the appservice guide should ask you to add a registration yaml
+file to the homeserver. In Synapse you would do this by adding the path to the
+homeserver.yaml, but in Conduit you can do this from within Matrix:
+
+First, go into the #admins room of your homeserver. The first person that
+registered on the homeserver automatically joins it. Then send a message into
+the room like this:
+
+    @conduit:your.server.name: register_appservice
+    ```
+    paste
+    the
+    contents
+    of
+    the
+    yaml
+    registration
+    here
+    ```
+
+You can confirm it worked by sending a message like this:
+`@conduit:your.server.name: list_appservices`
+
+The @conduit bot should answer with `Appservices (1): your-bridge`
+
+Then you are done. Conduit will send messages to the appservices and the
+appservice can send requests to the homeserver. You don't need to restart
+Conduit, but if it doesn't work, restarting while the appservice is running
+could help.

CROSS_COMPILE.md

@@ -1,13 +1,11 @@
 Install docker:
 
+```
 $ sudo apt install docker
 $ sudo usermod -aG docker $USER
-
-Then log out and back in.
-
+$ exec sudo su -l $USER
 $ sudo systemctl start docker
-
 $ cargo install cross
-$ cross build --release --features tls_vendored --target armv7-unknown-linux-musleabihf
-
+$ cross build --release --target armv7-unknown-linux-musleabihf
+```
 The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit

Cargo.toml

@@ -4,7 +4,7 @@ description = "A Matrix homeserver written in Rust"
 license = "Apache-2.0"
 authors = ["timokoesters <timo@koesters.xyz>"]
 homepage = "https://conduit.rs"
-repository = "https://git.koesters.xyz/timo/conduit"
+repository = "https://gitlab.com/famedly/conduit"
 readme = "README.md"
 version = "0.1.0"
 edition = "2018"
@@ -14,56 +14,79 @@ edition = "2018"
 [dependencies]
 # Used to handle requests
 # TODO: This can become optional as soon as proper configs are supported
-rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "1f1f44f336e5a172361fc1860461bb03667b1ed2", features = ["tls"] } # Used to handle requests
-#rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] }
+# rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "801e04bd5369eb39e126c75f6d11e1e9597304d8", features = ["tls"] } # Used to handle requests
+rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests
 
 # Used for matrix spec type definitions and helpers
-ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "e8882fe8142d7b55ed4c8ccc6150946945f9e237" }
-# ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "unstable-join" }
-# ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] }
-
-# Used when doing state resolution
-# state-res = { git = "https://github.com/timokoesters/state-res", branch = "spec-comp", features = ["unstable-pre-spec"] }
-state-res = { git = "https://github.com/ruma/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec", "gen-eventid"] }
-# state-res = { path = "../../state-res", features = ["unstable-pre-spec", "gen-eventid"] }
+#ruma = { git = "https://github.com/ruma/ruma", rev = "eb19b0e08a901b87d11b3be0890ec788cc760492", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+ruma = { git = "https://github.com/timokoesters/ruma", rev = "a2d93500e1dbc87e7032a3c74f3b2479a7f84e93", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+#ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 
 # Used for long polling and federation sender, should be the same as rocket::tokio
-tokio = { version = "0.2.23" }
+tokio = "1.8.2"
 # Used for storing data permanently
-sled = { version = "0.34.6", default-features = false }
-# Used for emitting log entries
-log = "0.4.11"
+sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true }
+#sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] }
+
+# Used for the http request / response body type for Ruma endpoints used with reqwest
+bytes = "1.0.1"
 # Used for rocket<->ruma conversions
-http = "0.2.1"
+http = "0.2.4"
 # Used to find data directory for default db path
-directories = "3.0.1"
-# Used for number types for ruma
-js_int = "0.1.9"
+directories = "3.0.2"
 # Used for ruma wrapper
-serde_json = { version = "1.0.60", features = ["raw_value"] }
+serde_json = { version = "1.0.64", features = ["raw_value"] }
+# Used for appservice registration files
+serde_yaml = "0.8.17"
 # Used for pdu definition
-serde = "1.0.117"
+serde = "1.0.126"
 # Used for secure identifiers
-rand = "0.7.3"
+rand = "0.8.4"
 # Used to hash passwords
 rust-argon2 = "0.8.3"
 # Used to send requests
-reqwest = "0.10.9"
+reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls-native-roots", "socks"] }
+# Custom TLS verifier
+rustls = { version = "0.19.1", features = ["dangerous_configuration"] }
+rustls-native-certs = "0.5.0"
+webpki = "0.21.0"
 # Used for conduit::Error type
-thiserror = "1.0.22"
+thiserror = "1.0.26"
 # Used to generate thumbnails for images
-image = { version = "0.23.12", default-features = false, features = ["jpeg", "png", "gif"] }
+image = { version = "0.23.14", default-features = false, features = ["jpeg", "png", "gif"] }
 # Used to encode server public key
 base64 = "0.13.0"
 # Used when hashing the state
-ring = "0.16.19"
+ring = "0.16.20"
 # Used when querying the SRV record of other servers
-trust-dns-resolver = "0.19.6"
+trust-dns-resolver = "0.20.3"
+# Used to find matching events for appservices
+regex = "1.5.4"
+# jwt jsonwebtokens
+jsonwebtoken = "7.2.0"
+# Performance measurements
+tracing = { version = "0.1.26", features = ["release_max_level_warn"] }
+opentelemetry = "0.15.0"
+tracing-subscriber = "0.2.19"
+tracing-opentelemetry = "0.14.0"
+tracing-flame = "0.1.0"
+opentelemetry-jaeger = "0.14.0"
+pretty_env_logger = "0.4.0"
+lru-cache = "0.1.2"
+rusqlite = { version = "0.25.3", optional = true, features = ["bundled"] }
+parking_lot = { version = "0.11.1", optional = true }
+crossbeam = { version = "0.8.1", optional = true }
+num_cpus = "1.13.0"
+threadpool = "1.8.1"
+heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true }
 
 [features]
-default = ["conduit_bin"]
+default = ["conduit_bin", "backend_sqlite"]
+backend_sled = ["sled"]
+backend_sqlite = ["sqlite"]
+backend_heed = ["heed", "crossbeam"]
+sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"]
 conduit_bin = [] # TODO: add rocket to this when it is optional
-tls_vendored = ["reqwest/native-tls-vendored"]
 
 [[bin]]
 name = "conduit"
@@ -73,3 +96,29 @@ required-features = ["conduit_bin"]
 [lib]
 name = "conduit"
 path = "src/lib.rs"
+
+[package.metadata.deb]
+name = "matrix-conduit"
+maintainer = "Paul van Tilburg <paul@luon.net>"
+copyright = "2020, Timo Kösters <timo@koesters.xyz>"
+license-file = ["LICENSE", "3"]
+depends = "$auto, ca-certificates"
+extended-description = """\
+A fast Matrix homeserver that is optimized for smaller, personal servers, \
+instead of a server that has high scalability."""
+section = "net"
+priority = "optional"
+assets = [
+  ["debian/README.Debian", "usr/share/doc/matrix-conduit/", "644"],
+  ["README.md", "usr/share/doc/matrix-conduit/", "644"],
+  ["target/release/conduit", "usr/sbin/matrix-conduit", "755"],
+]
+conf-files = [
+  "/etc/matrix-conduit/conduit.toml"
+]
+maintainer-scripts = "debian/"
+systemd-units = { unit-name = "matrix-conduit" }
+
+# For flamegraphs:
+#[profile.release]
+#debug = true

DEPLOY.md

@@ -0,0 +1,231 @@
+# Deploying Conduit
+
+## Getting help
+
+If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
+
+## Installing Conduit
+
+You may simply download the binary that fits your machine. Run `uname -m` to see
+what you need. Now copy the right url:
+
+| CPU Architecture     | GNU (Ubuntu, Debian, ArchLinux, ...)  | MUSL (Alpine, ... )     |
+| -------------------- | ------------------------------------- | ----------------------- |
+| x84_64 / amd64       | [Download][x84_64-gnu]                | [Download][x84_64-musl] |
+| armv7 (Raspberry Pi) | [Download][armv7-gnu]                 | -                       |
+| armv8 / aarch64      | [Download][armv8-gnu]                 | -                       |
+
+[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:cargo:x86_64-unknown-linux-gnu
+[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:cargo:x86_64-unknown-linux-musl
+[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:cargo:armv7-unknown-linux-gnueabihf
+[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:cargo:aarch64-unknown-linux-gnu
+
+```bash
+$ sudo wget -O /usr/local/bin/matrix-conduit <url>
+$ sudo chmod +x /usr/local/bin/matrix-conduit
+```
+
+Alternatively, you may compile the binary yourself using
+```bash
+$ cargo build --release
+```
+Note that this currently requires Rust 1.50.
+
+If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md).
+
+
+## Adding a Conduit user
+
+While Conduit can run as any user it is usually better to use dedicated users for different services.
+This also allows you to make sure that the file permissions are correctly set up.
+
+In Debian you can use this command to create a Conduit user:
+
+```
+sudo adduser --system conduit --no-create-home
+```
+
+## Setting up a systemd service
+
+Now we'll set up a systemd service for Conduit, so it's easy to start/stop
+Conduit and set it to autostart when your server reboots. Simply paste the
+default systemd service you can find below into
+`/etc/systemd/system/conduit.service`.
+
+```systemd
+[Unit]
+Description=Conduit Matrix Server
+After=network.target
+
+[Service]
+Environment="CONDUIT_CONFIG=/etc/matrix-conduit/conduit.toml"
+User=conduit
+Group=nogroup
+Restart=always
+ExecStart=/usr/local/bin/matrix-conduit
+
+[Install]
+WantedBy=multi-user.target
+```
+
+Finally, run
+
+```bash
+$ sudo systemctl daemon-reload
+```
+
+
+## Creating the Conduit configuration file
+
+Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment to read it. You need to change at least the server name.**
+
+```toml
+[global]
+# The server_name is the name of this server. It is used as a suffix for user
+# and room ids. Examples: matrix.org, conduit.rs
+# The Conduit server needs to be reachable at https://your.server.name/ on port
+# 443 (client-server) and 8448 (federation) OR you can create /.well-known
+# files to redirect requests. See
+# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
+# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server
+# for more information
+
+# YOU NEED TO EDIT THIS
+#server_name = "your.server.name"
+
+# This is the only directory where Conduit will save its data
+database_path = "/var/lib/matrix-conduit/conduit_db"
+
+# The port Conduit will be running on. You need to set up a reverse proxy in
+# your web server (e.g. apache or nginx), so all requests to /_matrix on port
+# 443 and 8448 will be forwarded to the Conduit instance running on this port
+port = 6167
+
+# Max size for uploads
+max_request_size = 20_000_000 # in bytes
+
+# Disabling registration means no new users will be able to register on this server
+allow_registration = false
+
+# Disable encryption, so no new encrypted rooms can be created
+# Note: existing rooms will continue to work
+allow_encryption = true
+allow_federation = true
+
+trusted_servers = ["matrix.org"]
+
+#max_concurrent_requests = 100 # How many requests Conduit sends to other servers at the same time
+#workers = 4 # default: cpu core count * 2
+
+address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy
+
+# The total amount of memory that the database will use.
+#db_cache_capacity_mb = 200
+```
+
+## Setting the correct file permissions
+
+As we are using a Conduit specific user we need to allow it to read the config.
+To do that you can run this command on Debian:
+
+```
+sudo chown -R conduit:nogroup /etc/matrix-conduit
+```
+
+If you use the default database path you also need to run this:
+
+```
+sudo mkdir -p /var/lib/matrix-conduit/conduit_db
+sudo chown -R conduit:nogroup /var/lib/matrix-conduit/conduit_db
+```
+
+
+## Setting up the Reverse Proxy
+
+This depends on whether you use Apache, Nginx or another web server.
+
+### Apache
+
+Create `/etc/apache2/sites-enabled/050-conduit.conf` and copy-and-paste this:
+
+```
+Listen 8448
+
+<VirtualHost *:443 *:8448>
+
+ServerName your.server.name # EDIT THIS
+
+AllowEncodedSlashes NoDecode
+ProxyPass /_matrix/ http://127.0.0.1:6167/_matrix/ nocanon
+ProxyPassReverse /_matrix/ http://127.0.0.1:6167/_matrix/
+
+Include /etc/letsencrypt/options-ssl-apache.conf
+SSLCertificateFile /etc/letsencrypt/live/your.server.name/fullchain.pem # EDIT THIS
+SSLCertificateKeyFile /etc/letsencrypt/live/your.server.name/privkey.pem # EDIT THIS
+</VirtualHost>
+```
+
+**You need to make some edits again.** When you are done, run
+
+```bash
+$ sudo systemctl reload apache2
+```
+
+
+### Nginx
+
+If you use Nginx and not Apache, add the following server section inside the
+http section of `/etc/nginx/nginx.conf`
+
+```
+server {
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+    listen 8448 ssl http2;
+    listen [::]:8448 ssl http2;
+    server_name your.server.name; # EDIT THIS
+    merge_slashes off;
+
+    location /_matrix/ {
+        proxy_pass http://127.0.0.1:6167$request_uri;
+        proxy_set_header Host $http_host;
+        proxy_buffering off;
+    }
+
+    ssl_certificate /etc/letsencrypt/live/your.server.name/fullchain.pem; # EDIT THIS
+    ssl_certificate_key /etc/letsencrypt/live/your.server.name/privkey.pem; # EDIT THIS
+    ssl_trusted_certificate /etc/letsencrypt/live/your.server.name/chain.pem; # EDIT THIS
+    include /etc/letsencrypt/options-ssl-nginx.conf;
+}
+```
+**You need to make some edits again.** When you are done, run
+
+```bash
+$ sudo systemctl reload nginx
+```
+
+
+## SSL Certificate
+
+The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this:
+
+```bash
+$ sudo certbot -d your.server.name
+```
+
+
+## You're done!
+
+Now you can start Conduit with:
+
+```bash
+$ sudo systemctl start conduit
+```
+
+Set it to start automatically when your system boots with:
+
+```bash
+$ sudo systemctl enable conduit
+```
+
+If you want to set up an appservice, take a look at the [Appservice Guide](APPSERVICES.md).

DEPLOY_FROM_SOURCE.md

@@ -1,103 +0,0 @@
-# Deploy from source
-
-## Prerequisites
-
-Make sure you have `libssl-dev` and `pkg-config` installed and the [rust toolchain](https://rustup.rs) is available on at least on user.
-
-
-## Install Conduit
-
-```bash
-$ sudo useradd -m conduit
-$ sudo -u conduit cargo install --git "https://git.koesters.xyz/timo/conduit.git"
-```
-
-
-## Setup systemd service
-
-In this guide, we set up a systemd service for Conduit, so it's easy to start, stop Conduit and set it to autostart when your server reboots. Paste the default systemd service below and configure it to fit your setup (in /etc/systemd/system/conduit.service).
-
-```systemd
-[Unit]
-Description=Conduit
-After=network.target
-
-[Service]
-Environment="ROCKET_SERVER_NAME=YOURSERVERNAME.HERE" # EDIT THIS
-
-Environment="ROCKET_PORT=14004" # Reverse proxy port
-
-#Environment="ROCKET_MAX_REQUEST_SIZE=20000000" # in bytes
-#Environment="ROCKET_REGISTRATION_DISABLED=true"
-#Environment="ROCKET_ENCRYPTION_DISABLED=true"
-#Environment="ROCKET_FEDERATION_ENABLED=true"
-#Environment="ROCKET_LOG=normal" # Detailed logging
-
-Environment="ROCKET_ENV=production"
-User=conduit
-Group=conduit
-Type=simple
-Restart=always
-ExecStart=/home/conduit/.cargo/bin/conduit
-
-[Install]
-WantedBy=multi-user.target
-```
-
-Finally, run
-```bash
-$ sudo systemctl daemon-reload
-```
-
-
-## Setup Reverse Proxy
-
-This depends on whether you use Apache, Nginx or something else. For Apache it looks like this (in /etc/apache2/sites-enabled/050-conduit.conf):
-```
-<VirtualHost *:443>
-
-ServerName conduit.koesters.xyz # EDIT THIS
-
-AllowEncodedSlashes NoDecode
-
-ServerAlias conduit.koesters.xyz # EDIT THIS
-
-ProxyPreserveHost On
-ProxyRequests off
-AllowEncodedSlashes NoDecode
-ProxyPass / http://localhost:14004/ nocanon
-ProxyPassReverse / http://localhost:14004/ nocanon
-
-Include /etc/letsencrypt/options-ssl-apache.conf
-
-# EDIT THESE:
-SSLCertificateFile /etc/letsencrypt/live/conduit.koesters.xyz/fullchain.pem
-SSLCertificateKeyFile /etc/letsencrypt/live/conduit.koesters.xyz/privkey.pem
-</VirtualHost>
-```
-
-Then run
-```bash
-$ sudo systemctl reload apache2
-```
-
-
-## SSL Certificate
-
-The easiest way to get an SSL certificate for the domain is to install `certbot` and run this:
-```bash
-$ sudo certbot -d conduit.koesters.xyz
-```
-
-
-## You're done!
-
-Now you can start Conduit with
-```bash
-$ sudo systemctl start conduit
-```
-
-and set it to start automatically when your system boots with
-```bash
-$ sudo systemctl enable conduit
-```

Dockerfile

@@ -5,7 +5,7 @@
 
 ##########################  BUILD IMAGE  ##########################
 # Alpine build image to build Conduit's statically compiled binary
-FROM alpine:3.12 as builder
+FROM alpine:3.14 as builder
 
 # Specifies if the local project is build or if Conduit gets build
 # from the official git repository. Defaults to the git repo.
@@ -13,11 +13,6 @@ ARG LOCAL=false
 # Specifies which revision/commit is build. Defaults to HEAD
 ARG GIT_REF=origin/master
 
-# Add 'edge'-repository to get Rust 1.45
-RUN sed -i \
-	-e 's|v3\.12|edge|' \
-	/etc/apk/repositories
-
 # Install packages needed for building all crates
 RUN apk add --no-cache \
         cargo \
@@ -30,17 +25,19 @@ COPY . .
 RUN if [[ $LOCAL == "true" ]]; then \
         cargo install --path . ; \
     else \
-        cargo install --git "https://github.com/timokoesters/conduit.git" --rev ${GIT_REF}; \
+        cargo install --git "https://gitlab.com/famedly/conduit.git" --rev ${GIT_REF}; \
     fi
 
 ########################## RUNTIME IMAGE ##########################
 # Create new stage with a minimal image for the actual
 # runtime image/container
-FROM alpine:3.12
+FROM alpine:3.14
 
 ARG CREATED
 ARG VERSION
-ARG GIT_REF=HEAD
+ARG GIT_REF=origin/master
+
+ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
 
 # Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md
 # including a custom label specifying the build command
@@ -52,15 +49,15 @@ LABEL org.opencontainers.image.created=${CREATED} \
       org.opencontainers.image.description="A Matrix homeserver written in Rust" \
       org.opencontainers.image.url="https://conduit.rs/" \
       org.opencontainers.image.revision=${GIT_REF} \
-      org.opencontainers.image.source="https://git.koesters.xyz/timo/conduit.git" \
+      org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
       org.opencontainers.image.licenses="Apache-2.0" \
       org.opencontainers.image.documentation="" \
       org.opencontainers.image.ref.name="" \
       org.label-schema.docker.build="docker build . -t matrixconduit/matrix-conduit:latest --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)" \
       maintainer="Weasy666"
 
-# Standard port on which Rocket launches
-EXPOSE 8000
+# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose.
+EXPOSE 6167
 
 # Copy config files from context and the binary from
 # the "builder" stage to the current stage into folder
@@ -84,11 +81,11 @@ RUN apk add --no-cache \
         curl \
         libgcc
 
-# Create a volume for the database, to persist its contents
-VOLUME ["/srv/conduit/.local/share/conduit"]
-
 # Test if Conduit is still alive, uses the same endpoint as Element
-HEALTHCHECK --start-period=2s CMD curl --fail -s http://localhost:8000/_matrix/client/versions || curl -k --fail -s https://localhost:8000/_matrix/client/versions || exit 1
+HEALTHCHECK --start-period=5s \
+    CMD curl --fail -s "http://localhost:$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')/_matrix/client/versions" || \
+        curl -k --fail -s "https://localhost:$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')/_matrix/client/versions" || \
+        exit 1
 
 # Set user to www-data
 USER www-data

README.md

@@ -1,40 +1,20 @@
 # Conduit
 ### A Matrix homeserver written in Rust
 
-[![Liberapay](https://img.shields.io/liberapay/receives/timokoesters?logo=liberapay)](https://liberapay.com/timokoesters)
-[![Matrix](https://img.shields.io/matrix/conduit:conduit.rs?server_fqdn=conduit.koesters.xyz)](https://matrix.to/#/#conduit:matrix.org)
-
 #### What is the goal?
 
 A fast Matrix homeserver that's easy to set up and just works. You can install
 it on a mini-computer like the Raspberry Pi to host Matrix for your family,
 friends or company.
 
+
 #### Can I try it out?
 
 Yes! Just open a Matrix client (<https://app.element.io> or Element Android for
 example) and register on the `https://conduit.koesters.xyz` homeserver.
 
-#### How can I deploy my own?
 
-##### From source
-
-Clone the repo, build it with `cargo build --release` and call the binary
-(target/release/conduit) from somewhere like a systemd script. [Read
-more](DEPLOY_FROM_SOURCE.md)
-
-##### Using Docker
-
-Pull and run the docker image with
-
-``` bash
-docker pull matrixconduit/matrix-conduit:latest
-docker run -d -p 8448:8000 -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
-```
-
-Or build and run it with docker or docker-compose. [Read more](docker/README.md)
-
-#### What is it build on?
+#### What is it built on?
 
 - [Ruma](https://www.ruma.io): Useful structures for endpoint requests and
   responses that can be (de)serialized
@@ -42,26 +22,89 @@ Or build and run it with docker or docker-compose. [Read more](docker/README.md)
   good performance
 - [Rocket](https://rocket.rs): A flexible web framework
 
-#### What are the biggest things still missing?
 
-- Appservices (Bridges and Bots)
-- Most federation features (invites, e2ee)
-- Push notifications on mobile
-- Notification settings
+#### What is the current status?
+
+Conduit can already be used chat with other users on Conduit, chat with users
+from other Matrix servers and even to chat with users on other platforms using
+appservices. When chatting with users on the same Conduit server, everything
+should work assuming you use a compatible client.
+
+**You should not join Matrix rooms without asking the admins first.** We do not
+know whether Conduit is safe for general use yet, so you should assume there is
+some chance that it breaks rooms permanently for all participating users. We
+are not aware of such a bug today, but we would like to do more testing.
+
+There are still a few important features missing:
+
+- Database stability (currently you might have to do manual upgrades or even wipe the db for new versions)
+- Edge cases for end-to-end encryption over federation
+- Typing and presence over federation
 - Lots of testing
 
-Also check out the [milestones](https://git.koesters.xyz/timo/conduit/milestones).
+Check out the [Conduit 1.0 Release Milestone](https://gitlab.com/famedly/conduit/-/milestones/3).
+
+
+#### How can I deploy my own?
+
+##### Deploy
+
+Download or compile a Conduit binary, set up the config and call it from somewhere like a systemd script. [Read
+more](DEPLOY.md)
+
+If you want to connect an Appservice to Conduit, take a look at the [Appservice Guide](APPSERVICES.md).
+
+##### Deploy using a Debian package
+
+You need to have the `deb` helper command installed that creates Debian packages from Cargo projects (see [cargo-deb](https://github.com/mmstick/cargo-deb/) for more info):
+
+```shell
+$ cargo install cargo-deb
+```
+
+Then, you can create and install a Debian package at a whim:
+
+```shell
+$ cargo deb
+$ dpkg -i target/debian/matrix-conduit_0.1.0_amd64.deb
+```
+
+This will build, package, install, configure and start Conduit. [Read more](debian/README.Debian).
+
+Note that `cargo deb` supports [cross-compilation](https://github.com/mmstick/cargo-deb/#cross-compilation) too!
+Official Debian packages will follow once Conduit starts to have stable releases.
+
+##### Deploy using Docker
+
+Pull and run the docker image with
+
+``` bash
+docker pull matrixconduit/matrix-conduit:latest
+docker run -d -p 8448:8000 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
+```
+
+> <b>Note:</b> You also need to supply a `conduit.toml` config file, you can find an example [here](./conduit-example.toml).
+> Or you can pass in `-e CONDUIT_CONFIG=""` and configure Conduit purely with env vars.
+
+Or build and run it with docker or docker-compose. [Read more](docker/README.md)
+
 
 #### How can I contribute?
 
 1. Look for an issue you would like to work on and make sure it's not assigned
    to other users
 2. Ask someone to assign the issue to you (comment on the issue or chat in
-   #conduit:matrix.org)
-3. Fork the repo and work on the issue. #conduit:matrix.org is happy to help :)
-4. Submit a PR
+   #conduit:nordgedanken.dev)
+3. Fork the repo and work on the issue. #conduit:nordgedanken.dev is happy to help :)
+4. Submit a MR
 
 #### Donate
 
 Liberapay: <https://liberapay.com/timokoesters/>\
 Bitcoin: `bc1qnnykf986tw49ur7wx9rpw2tevpsztvar5x8w4n`
+
+
+#### Logo
+
+Lightning Bolt Logo: https://github.com/mozilla/fxemoji/blob/gh-pages/svgs/nature/u26A1-bolt.svg \
+Logo License: https://github.com/mozilla/fxemoji/blob/gh-pages/LICENSE.md

Rocket-example.toml

@@ -1,31 +0,0 @@
-[global]
-# The name of this server
-# Note: If server name != hostname, you need a .well-known file for federation
-# to work
-server_name = "your.server.name"
-
-port = 14004
-
-# Max size for uploads
-#max_request_size = 20_000_000 # in bytes, ~20 MB
-
-# Disable registration. No new users will be able to register on this server
-#registration_disabled = true
-
-# Disable encryption, so no new encrypted rooms can be created
-# Note: existing rooms will continue to work
-#encryption_disabled = true
-
-#federation_enabled = true
-
-# Default path is in this user's data
-#database_path = "/home/timo/MyConduitServer"
-
-# You should probably leave this at 0.0.0.0
-address = "0.0.0.0"
-
-# TLS support
-# Note: Not necessary when using a reverse proxy:
-#[global.tls]
-#certs = "/etc/letsencrypt/live/your.server.name/fullchain.pem"
-#key = "/etc/letsencrypt/live/your.server.name/privkey.pem"

conduit-example.toml

@@ -0,0 +1,47 @@
+[global]
+# The server_name is the name of this server. It is used as a suffix for user
+# and room ids. Examples: matrix.org, conduit.rs
+# The Conduit server needs to be reachable at https://your.server.name/ on port
+# 443 (client-server) and 8448 (federation) OR you can create /.well-known
+# files to redirect requests. See
+# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
+# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server
+# for more information
+
+# YOU NEED TO EDIT THIS
+#server_name = "your.server.name"
+
+# This is the only directory where Conduit will save its data
+database_path = "/var/lib/conduit/"
+
+# The port Conduit will be running on. You need to set up a reverse proxy in
+# your web server (e.g. apache or nginx), so all requests to /_matrix on port
+# 443 and 8448 will be forwarded to the Conduit instance running on this port
+port = 6167
+
+# Max size for uploads
+max_request_size = 20_000_000 # in bytes
+
+# Disable registration. No new users will be able to register on this server
+#allow_registration = false
+
+# Disable encryption, so no new encrypted rooms can be created
+# Note: existing rooms will continue to work
+#allow_encryption = false
+#allow_federation = false
+
+# Enable jaeger to support monitoring and troubleshooting through jaeger
+#allow_jaeger = false
+
+trusted_servers = ["matrix.org"]
+
+#max_concurrent_requests = 100 # How many requests Conduit sends to other servers at the same time
+#log = "info,state_res=warn,rocket=off,_=off,sled=off"
+#workers = 4 # default: cpu core count * 2
+
+address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy
+
+proxy = "none" # more examples can be found at src/database/proxy.rs:6
+
+# The total amount of memory that the database will use.
+#db_cache_capacity_mb = 200

debian/README.Debian

@@ -0,0 +1,29 @@
+Conduit for Debian
+==================
+
+Configuration
+-------------
+
+When installed, Debconf generates the configuration of the homeserver
+(host)name, the address and port it listens on. This configuration ends up in
+/etc/matrix-conduit/conduit.toml.
+
+You can tweak more detailed settings by uncommenting and setting the variables
+in /etc/matrix-conduit/conduit.toml. This involves settings such as the maximum
+file size for download/upload, enabling federation, etc.
+
+Running
+-------
+
+The package uses the matrix-conduit.service systemd unit file to start and
+stop Conduit. It loads the configuration file mentioned above to set up the
+environment before running the server.
+
+This package assumes by default that Conduit will be placed behind a reverse
+proxy such as Apache or nginx. This default deployment entails just listening
+on 127.0.0.1 and the free port 6167 and is reachable via a client using the URL
+http://localhost:6167.
+
+At a later stage this packaging may support also setting up TLS and running
+stand-alone.  In this case, however, you need to set up some certificates and
+renewal, for it to work properly.

debian/config

@@ -0,0 +1,17 @@
+#!/bin/sh
+set -e
+
+# Source debconf library.
+. /usr/share/debconf/confmodule
+
+# Ask for the Matrix homeserver name, address and port.
+db_input high matrix-conduit/hostname || true
+db_go
+
+db_input low matrix-conduit/address || true
+db_go
+
+db_input medium matrix-conduit/port || true
+db_go
+
+exit 0

debian/matrix-conduit.service

@@ -0,0 +1,46 @@
+[Unit]
+Description=Conduit Matrix homeserver
+After=network.target
+
+[Service]
+User=_matrix-conduit
+Group=_matrix-conduit
+Type=simple
+
+AmbientCapabilities=
+CapabilityBoundingSet=
+LockPersonality=yes
+MemoryDenyWriteExecute=yes
+NoNewPrivileges=yes
+ProtectClock=yes
+ProtectControlGroups=yes
+ProtectHome=yes
+ProtectHostname=yes
+ProtectKernelLogs=yes
+ProtectKernelModules=yes
+ProtectKernelTunables=yes
+ProtectSystem=strict
+PrivateDevices=yes
+PrivateMounts=yes
+PrivateTmp=yes
+PrivateUsers=yes
+RemoveIPC=yes
+RestrictAddressFamilies=AF_INET AF_INET6
+RestrictNamespaces=yes
+RestrictRealtime=yes
+RestrictSUIDSGID=yes
+SystemCallArchitectures=native
+SystemCallFilter=@system-service
+SystemCallErrorNumber=EPERM
+StateDirectory=matrix-conduit
+
+Environment="CONDUIT_CONFIG=/etc/matrix-conduit/conduit.toml"
+
+ExecStart=/usr/sbin/matrix-conduit
+Restart=on-failure
+RestartSec=10
+StartLimitInterval=1m
+StartLimitBurst=5
+
+[Install]
+WantedBy=multi-user.target

debian/postinst

@@ -0,0 +1,87 @@
+#!/bin/sh
+set -e
+
+. /usr/share/debconf/confmodule
+
+CONDUIT_CONFIG_PATH=/etc/matrix-conduit
+CONDUIT_CONFIG_FILE="${CONDUIT_CONFIG_PATH}/conduit.toml"
+CONDUIT_DATABASE_PATH=/var/lib/matrix-conduit/conduit_db
+
+case "$1" in
+  configure)
+    # Create the `_matrix-conduit` user if it does not exist yet.
+    if ! getent passwd _matrix-conduit > /dev/null ; then
+      echo 'Adding system user for the Conduit Matrix homeserver' 1>&2
+      adduser --system --group --quiet \
+        --home "$CONDUIT_DATABASE_PATH" \
+        --disabled-login \
+        --force-badname \
+        _matrix-conduit
+    fi
+
+    # Create the database path if it does not exist yet.
+    if [ ! -d "$CONDUIT_DATABASE_PATH" ]; then
+      mkdir -p "$CONDUIT_DATABASE_PATH"
+      chown _matrix-conduit "$CONDUIT_DATABASE_PATH"
+    fi
+
+    if [ ! -e "$CONDUIT_CONFIG_FILE" ]; then
+      # Write the debconf values in the config.
+      db_get matrix-conduit/hostname
+      CONDUIT_SERVER_NAME="$RET"
+      db_get matrix-conduit/address
+      CONDUIT_ADDRESS="$RET"
+      db_get matrix-conduit/port
+      CONDUIT_PORT="$RET"
+      mkdir -p "$CONDUIT_CONFIG_PATH"
+      cat > "$CONDUIT_CONFIG_FILE" << EOF
+[global]
+# The server_name is the name of this server. It is used as a suffix for user
+# and room ids. Examples: matrix.org, conduit.rs
+# The Conduit server needs to be reachable at https://your.server.name/ on port
+# 443 (client-server) and 8448 (federation) OR you can create /.well-known
+# files to redirect requests. See
+# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
+# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server
+# for more information.
+server_name = "${CONDUIT_SERVER_NAME}"
+
+# This is the only directory where Conduit will save its data.
+database_path = "${CONDUIT_DATABASE_PATH}"
+
+# The address Conduit will be listening on.
+# By default the server listens on address 0.0.0.0. Change this to 127.0.0.1 to
+# only listen on the localhost when using a reverse proxy.
+address = "${CONDUIT_ADDRESS}"
+
+# The port Conduit will be running on. You need to set up a reverse proxy in
+# your web server (e.g. apache or nginx), so all requests to /_matrix on port
+# 443 and 8448 will be forwarded to the Conduit instance running on this port.
+port = ${CONDUIT_PORT}
+
+# Max size for uploads
+max_request_size = 20_000_000 # in bytes
+
+# Disable registration. No new users will be able to register on this server.
+#allow_registration = false
+
+# Disable encryption, so no new encrypted rooms can be created.
+# Note: Existing rooms will continue to work.
+#allow_encryption = false
+#allow_federation = false
+
+# Enable jaeger to support monitoring and troubleshooting through jaeger.
+#allow_jaeger = false
+
+#max_concurrent_requests = 100 # How many requests Conduit sends to other servers at the same time
+#log = "info,state_res=warn,rocket=off,_=off,sled=off"
+#workers = 4 # default: cpu core count * 2
+
+# The total amount of memory that the database will use.
+#db_cache_capacity_mb = 200
+EOF
+    fi
+    ;;
+esac
+
+#DEBHELPER#

debian/postrm

@@ -0,0 +1,27 @@
+#!/bin/sh
+set -e
+
+. /usr/share/debconf/confmodule
+
+CONDUIT_CONFIG_PATH=/etc/matrix-conduit
+CONDUIT_DATABASE_PATH=/var/lib/matrix-conduit
+
+case $1 in
+  purge)
+    # Remove debconf changes from the db
+    db_purge
+
+    # Per https://www.debian.org/doc/debian-policy/ch-files.html#behavior
+    # "configuration files must be preserved when the package is removed, and
+    #  only deleted when the package is purged."
+    if [ -d "$CONDUIT_CONFIG_PATH" ]; then
+      rm -r "$CONDUIT_CONFIG_PATH"
+    fi
+
+    if [ -d "$CONDUIT_DATABASE_PATH" ]; then
+      rm -r "$CONDUIT_DATABASE_PATH"
+    fi
+    ;;
+esac
+
+#DEBHELPER#

debian/templates

@@ -0,0 +1,21 @@
+Template: matrix-conduit/hostname
+Type: string
+Default: localhost
+Description: The server (host)name of the Matrix homeserver
+ This is the hostname the homeserver will be reachable at via a client.
+ .
+ If set to "localhost", you can connect with a client locally and clients
+ from other hosts and also other homeservers will not be able to reach you!
+
+Template: matrix-conduit/address
+Type: string
+Default: 127.0.0.1
+Description: The listen address of the Matrix homeserver
+ This is the address the homeserver will listen on. Leave it set to 127.0.0.1
+ when using a reverse proxy.
+
+Template: matrix-conduit/port
+Type: string
+Default: 6167
+Description: The port of the Matrix homeserver
+ This port is most often just accessed by a reverse proxy.

docker-compose.yml

@@ -3,7 +3,7 @@ version: '3'
 
 services:
     homeserver:
-        ### If you already built the Conduit image with 'docker build' or want to use the Docker Hub image,
+        ### If you already built the Conduit image with 'docker build' or want to use a registry image,
         ### then you are ready to go.
         image: matrixconduit/matrix-conduit:latest
         ### If you want to build a fresh image from the sources, then comment the image line and uncomment the
@@ -12,36 +12,41 @@ services:
         # build:
         #     context: .
         #     args:
-        #         CREATED:
-        #         VERSION:
+        #         CREATED: '2021-03-16T08:18:27Z'
+        #         VERSION: '0.1.0'
         #         LOCAL: 'false'
-        #         GIT_REF: HEAD
+        #         GIT_REF: origin/master
         restart: unless-stopped
         ports:
-            - 8448:8000
+            - 8448:6167
         volumes:
             - db:/srv/conduit/.local/share/conduit
-            ### Uncomment if you want to use Rocket.toml to configure Conduit
-            ### Note: Set env vars will override Rocket.toml values
-            # - ./Rocket.toml:/srv/conduit/Rocket.toml
+            ### Uncomment if you want to use conduit.toml to configure Conduit
+            ### Note: Set env vars will override conduit.toml values
+            # - ./conduit.toml:/srv/conduit/conduit.toml
         environment:
-            ROCKET_SERVER_NAME: localhost:8000  # replace with your own name
+            CONDUIT_SERVER_NAME: localhost:6167  # replace with your own name
+            CONDUIT_TRUSTED_SERVERS: '["matrix.org"]'
             ### Uncomment and change values as desired
-            # ROCKET_LOG: normal  # Available levels are: off, debug, normal, critical
-            # ROCKET_PORT: 8000
-            # ROCKET_REGISTRATION_DISABLED: 'true'
-            # ROCKET_ENCRYPTION_DISABLED: 'true'
-            # ROCKET_FEDERATION_ENABLED: 'true'
-            # ROCKET_DATABASE_PATH: /srv/conduit/.local/share/conduit
-            # ROCKET_WORKERS: 10
-            # ROCKET_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB
+            # CONDUIT_ADDRESS: 0.0.0.0
+            # CONDUIT_PORT: 6167
+            # CONDUIT_CONFIG: '/srv/conduit/conduit.toml'  # if you want to configure purely by env vars, set this to an empty string ''
+            # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging
+            # CONDUIT_LOG: info  # default is: "info,rocket=off,_=off,sled=off"
+            # CONDUIT_ALLOW_JAEGER: 'false'
+            # CONDUIT_ALLOW_REGISTRATION : 'false'
+            # CONDUIT_ALLOW_ENCRYPTION: 'false'
+            # CONDUIT_ALLOW_FEDERATION: 'false'
+            # CONDUIT_DATABASE_PATH: /srv/conduit/.local/share/conduit
+            # CONDUIT_WORKERS: 10
+            # CONDUIT_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB
 
     ### Uncomment if you want to use your own Element-Web App.
     ### Note: You need to provide a config.json for Element and you also need a second
     ###       Domain or Subdomain for the communication between Element and Conduit
     ### Config-Docs: https://github.com/vector-im/element-web/blob/develop/docs/config.md
     # element-web:
-    #     image: vectorim/riot-web:latest
+    #     image: vectorim/element-web:latest
     #     restart: unless-stopped
     #     ports:
     #         - 8009:80
@@ -51,4 +56,4 @@ services:
     #         - homeserver
 
 volumes:
-    db:
+  db:

docker/README.md

@@ -17,13 +17,13 @@ The Dockerfile includes a few build arguments that should be supplied when build
 ARG LOCAL=false
 ARG CREATED
 ARG VERSION
-ARG GIT_REF=HEAD
+ARG GIT_REF=origin/master
 ```
 
 - **CREATED:** Date and time as string (date-time as defined by RFC 3339). Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.created`. Supply by it like this `$(date -u +'%Y-%m-%dT%H:%M:%SZ')`
 - **VERSION:** The SemVer version of Conduit, which is in the image. Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.version`. If you have a `Cargo.toml` in your build context, you can get it with `$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)`
 - **LOCAL:** *(Optional)* A boolean value, specifies if the local build context should be used, or if the official repository will be cloned. If not supplied with the build command, it will default to `false`.
-- **GIT_REF:** *(Optional)* A git ref, like `HEAD` or a commit ID. The supplied ref will be used to create the Open Container Initiative compliant label `org.opencontainers.image.revision` and will be the ref that is cloned from the repository when not building from the local context. If not supplied with the build command, it will default to `HEAD`.
+- **GIT_REF:** *(Optional)* A git ref, like `HEAD` or a commit ID. The supplied ref will be used to create the Open Container Initiative compliant label `org.opencontainers.image.revision` and will be the ref that is cloned from the repository when not building from the local context. If not supplied with the build command, it will default to `origin/master`.
 
 To build the image you can use the following command
 
@@ -40,10 +40,12 @@ which also will tag the resulting image as `matrixconduit/matrix-conduit:latest`
 After building the image you can simply run it with
 
 ``` bash
-docker run -d -p 8448:8000 -v db:/srv/conduit/.local/share/conduit -e ROCKET_SERVER_NAME="localhost:8000" matrixconduit/matrix-conduit:latest
+docker run -d -p 8448:8000 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
 ```
 
-For detached mode, you also need to use the `-d` flag. You can pass in more env vars as are shown here, for an overview of possible values, you can take a look at the `docker-compose.yml` file.
+For detached mode, you also need to use the `-d` flag. You also need to supply a `conduit.toml` config file, you can find an example [here](../conduit-example.toml).
+You can pass in different env vars to change config values on the fly. You can even configure Conduit completely by using env vars, but for that you need
+too pass `-e CONDUIT_CONFIG=""` into your container. For an overview of possible values, please take a look at the `docker-compose.yml` file.
 If you just want to test Conduit for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it.
 
 

docker/ci-binaries-packaging.Dockerfile

@@ -0,0 +1,70 @@
+# ---------------------------------------------------------------------------------------------------------
+# This Dockerfile is intended to be built as part of Conduit's CI pipeline.
+# It does not build Conduit in Docker, but just copies the matching build artifact from the build job.
+# As a consequence, this is not a multiarch capable image. It always expects and packages a x86_64 binary.
+#
+# It is mostly based on the normal Conduit Dockerfile, but adjusted in a few places to maximise caching.
+# Credit's for the original Dockerfile: Weasy666.
+# ---------------------------------------------------------------------------------------------------------
+
+FROM alpine:3.14
+
+ARG CREATED
+ARG VERSION
+ARG GIT_REF
+
+ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
+
+# Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md
+# including a custom label specifying the build command
+LABEL org.opencontainers.image.created=${CREATED} \
+      org.opencontainers.image.authors="Conduit Contributors" \
+      org.opencontainers.image.title="Conduit" \
+      org.opencontainers.image.version=${VERSION} \
+      org.opencontainers.image.vendor="Conduit Contributors" \
+      org.opencontainers.image.description="A Matrix homeserver written in Rust" \
+      org.opencontainers.image.url="https://conduit.rs/" \
+      org.opencontainers.image.revision=${GIT_REF} \
+      org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
+      org.opencontainers.image.licenses="Apache-2.0" \
+      org.opencontainers.image.documentation="" \
+      org.opencontainers.image.ref.name=""
+
+# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose.
+EXPOSE 6167
+
+# create data folder for database
+RUN mkdir -p /srv/conduit/.local/share/conduit
+
+# Add www-data user and group with UID 82, as used by alpine
+# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
+RUN set -x ; \
+    addgroup -Sg 82 www-data 2>/dev/null ; \
+    adduser -S -D -H -h /srv/conduit -G www-data -g www-data www-data 2>/dev/null ; \
+    addgroup www-data www-data 2>/dev/null && exit 0 ; exit 1
+
+# Change ownership of Conduit files to www-data user and group
+RUN chown -cR www-data:www-data /srv/conduit
+
+# Install packages needed to run Conduit
+RUN apk add --no-cache \
+        ca-certificates \
+        curl \
+        libgcc
+
+# Test if Conduit is still alive, uses the same endpoint as Element
+HEALTHCHECK --start-period=5s \
+    CMD curl --fail -s "http://localhost:$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')/_matrix/client/versions" || \
+        curl -k --fail -s "https://localhost:$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')/_matrix/client/versions" || \
+        exit 1
+
+# Set user to www-data
+USER www-data
+# Set container home directory
+WORKDIR /srv/conduit
+# Run Conduit
+ENTRYPOINT [ "/srv/conduit/conduit" ]
+
+
+# Copy the Conduit binary into the image at the latest possible moment to maximise caching:
+COPY ./conduit-x86_64-unknown-linux-musl /srv/conduit/conduit

docker/docker-compose.traefik.yml

@@ -12,35 +12,41 @@ services:
         # build:
         #     context: .
         #     args:
-        #         CREATED:
-        #         VERSION:
+        #         CREATED: '2021-03-16T08:18:27Z'
+        #         VERSION: '0.1.0'
         #         LOCAL: 'false'
-        #         GIT_REF: HEAD
+        #         GIT_REF: origin/master
         restart: unless-stopped
         volumes:
             - db:/srv/conduit/.local/share/conduit
-            ### Uncomment if you want to use Rocket.toml to configure Conduit
-            ### Note: Set env vars will override Rocket.toml values
-            # - ./Rocket.toml:/srv/conduit/Rocket.toml
+            ### Uncomment if you want to use conduit.toml to configure Conduit
+            ### Note: Set env vars will override conduit.toml values
+            # - ./conduit.toml:/srv/conduit/conduit.toml
         networks:
             - proxy
         environment:
-            ROCKET_SERVER_NAME: localhost:8000  # replace with your own name
+            CONDUIT_SERVER_NAME: localhost:6167  # replace with your own name
+            CONDUIT_TRUSTED_SERVERS: '["matrix.org"]'
             ### Uncomment and change values as desired
-            # ROCKET_LOG: normal  # Available levels are: off, debug, normal, critical
-            # ROCKET_PORT: 8000
-            # ROCKET_REGISTRATION_DISABLED: 'true'
-            # ROCKET_ENCRYPTION_DISABLED: 'true'
-            # ROCKET_DATABASE_PATH: /srv/conduit/.local/share/conduit
-            # ROCKET_WORKERS: 10
-            # ROCKET_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB
+            # CONDUIT_ADDRESS: 0.0.0.0
+            # CONDUIT_PORT: 6167
+            # CONDUIT_CONFIG: '/srv/conduit/conduit.toml'  # if you want to configure purely by env vars, set this to an empty string ''
+            # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging
+            # CONDUIT_LOG: info  # default is: "info,rocket=off,_=off,sled=off"
+            # CONDUIT_ALLOW_JAEGER: 'false'
+            # CONDUIT_ALLOW_REGISTRATION : 'false'
+            # CONDUIT_ALLOW_ENCRYPTION: 'false'
+            # CONDUIT_ALLOW_FEDERATION: 'false'
+            # CONDUIT_DATABASE_PATH: /srv/conduit/.local/share/conduit
+            # CONDUIT_WORKERS: 10
+            # CONDUIT_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB
 
     ### Uncomment if you want to use your own Element-Web App.
     ### Note: You need to provide a config.json for Element and you also need a second
     ###       Domain or Subdomain for the communication between Element and Conduit
     ### Config-Docs: https://github.com/vector-im/element-web/blob/develop/docs/config.md
     # element-web:
-    #     image: vectorim/riot-web:latest
+    #     image: vectorim/element-web:latest
     #     restart: unless-stopped
     #     volumes:
     #         - ./element_config.json:/app/config.json
@@ -53,7 +59,7 @@ volumes:
     db:
 
 networks:
-    # This is the network Traefik listens to, if you network has a different
+    # This is the network Traefik listens to, if your network has a different
     # name, don't forget to change it here and in the docker-compose.override.yml
     proxy:
         external: true

rust-toolchain

@@ -1 +1 @@
-1.47.0
+1.52

rustfmt.toml

@@ -1 +1,2 @@
-merge_imports = true
+unstable_features = true
+imports_granularity="Crate"

src/appservice_server.rs

@@ -0,0 +1,90 @@
+use crate::{utils, Error, Result};
+use bytes::BytesMut;
+use ruma::api::{IncomingResponse, OutgoingRequest, SendAccessToken};
+use std::{
+    convert::{TryFrom, TryInto},
+    fmt::Debug,
+    mem,
+    time::Duration,
+};
+use tracing::warn;
+
+pub async fn send_request<T: OutgoingRequest>(
+    globals: &crate::database::globals::Globals,
+    registration: serde_yaml::Value,
+    request: T,
+) -> Result<T::IncomingResponse>
+where
+    T: Debug,
+{
+    let destination = registration.get("url").unwrap().as_str().unwrap();
+    let hs_token = registration.get("hs_token").unwrap().as_str().unwrap();
+
+    let mut http_request = request
+        .try_into_http_request::<BytesMut>(&destination, SendAccessToken::IfRequired(""))
+        .unwrap()
+        .map(|body| body.freeze());
+
+    let mut parts = http_request.uri().clone().into_parts();
+    let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned();
+    let symbol = if old_path_and_query.contains('?') {
+        "&"
+    } else {
+        "?"
+    };
+
+    parts.path_and_query = Some(
+        (old_path_and_query + symbol + "access_token=" + hs_token)
+            .parse()
+            .unwrap(),
+    );
+    *http_request.uri_mut() = parts.try_into().expect("our manipulation is always valid");
+
+    let mut reqwest_request = reqwest::Request::try_from(http_request)
+        .expect("all http requests are valid reqwest requests");
+
+    *reqwest_request.timeout_mut() = Some(Duration::from_secs(30));
+
+    let url = reqwest_request.url().clone();
+    let mut response = globals.reqwest_client().execute(reqwest_request).await?;
+
+    // reqwest::Response -> http::Response conversion
+    let status = response.status();
+    let mut http_response_builder = http::Response::builder()
+        .status(status)
+        .version(response.version());
+    mem::swap(
+        response.headers_mut(),
+        http_response_builder
+            .headers_mut()
+            .expect("http::response::Builder is usable"),
+    );
+
+    let body = response.bytes().await.unwrap_or_else(|e| {
+        warn!("server error: {}", e);
+        Vec::new().into()
+    }); // TODO: handle timeout
+
+    if status != 200 {
+        warn!(
+            "Appservice returned bad response {} {}\n{}\n{:?}",
+            destination,
+            status,
+            url,
+            utils::string_from_bytes(&body)
+        );
+    }
+
+    let response = T::IncomingResponse::try_from_http_response(
+        http_response_builder
+            .body(body)
+            .expect("reqwest body is valid http body"),
+    );
+    response.map_err(|_| {
+        warn!(
+            "Appservice returned invalid response bytes {}\n{}",
+            destination, url
+        );
+        Error::BadServerResponse("Server returned bad response.")
+    })
+}

src/client_server/account.rs

@@ -1,8 +1,11 @@
-use std::{collections::BTreeMap, convert::TryInto};
+use std::{
+    collections::BTreeMap,
+    convert::{TryFrom, TryInto},
+    sync::Arc,
+};
 
-use super::{State, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH};
-use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma};
-use log::info;
+use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH};
+use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma};
 use ruma::{
     api::client::{
         error::ErrorKind,
@@ -11,6 +14,7 @@ use ruma::{
                 change_password, deactivate, get_username_availability, register, whoami,
                 ThirdPartyIdRemovalStatus,
             },
+            contact::get_contacts,
             uiaa::{AuthFlow, UiaaInfo},
         },
     },
@@ -21,8 +25,10 @@ use ruma::{
         },
         EventType,
     },
-    RoomAliasId, RoomId, RoomVersionId, UserId,
+    identifiers::RoomName,
+    push, RoomAliasId, RoomId, RoomVersionId, UserId,
 };
+use tracing::info;
 
 use register::RegistrationKind;
 #[cfg(feature = "conduit_bin")]
@@ -40,8 +46,9 @@ const GUEST_NAME_LENGTH: usize = 10;
     feature = "conduit_bin",
     get("/_matrix/client/r0/register/available", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_register_available_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_username_availability::Request<'_>>,
 ) -> ConduitResult<get_username_availability::Response> {
     // Validate user id
@@ -82,11 +89,12 @@ pub async fn get_register_available_route(
     feature = "conduit_bin",
     post("/_matrix/client/r0/register", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn register_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<register::Request<'_>>,
 ) -> ConduitResult<register::Response> {
-    if db.globals.registration_disabled() {
+    if !db.globals.allow_registration() && !body.from_appservice {
         return Err(Error::BadRequest(
             ErrorKind::Forbidden,
             "Registration has been disabled.",
@@ -139,18 +147,34 @@ pub async fn register_route(
         auth_error: None,
     };
 
-    if let Some(auth) = &body.auth {
-        let (worked, uiaainfo) =
-            db.uiaa
-                .try_auth(&user_id, "".into(), auth, &uiaainfo, &db.users, &db.globals)?;
-        if !worked {
+    if !body.from_appservice {
+        if let Some(auth) = &body.auth {
+            let (worked, uiaainfo) = db.uiaa.try_auth(
+                &UserId::parse_with_server_name("", db.globals.server_name())
+                    .expect("we know this is valid"),
+                "".into(),
+                auth,
+                &uiaainfo,
+                &db.users,
+                &db.globals,
+            )?;
+            if !worked {
+                return Err(Error::Uiaa(uiaainfo));
+            }
+        // Success!
+        } else if let Some(json) = body.json_body {
+            uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
+            db.uiaa.create(
+                &UserId::parse_with_server_name("", db.globals.server_name())
+                    .expect("we know this is valid"),
+                "".into(),
+                &uiaainfo,
+                &json,
+            )?;
             return Err(Error::Uiaa(uiaainfo));
+        } else {
+            return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
         }
-    // Success!
-    } else {
-        uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&user_id, "".into(), &uiaainfo)?;
-        return Err(Error::Uiaa(uiaainfo));
     }
 
     if missing_username {
@@ -163,12 +187,16 @@ pub async fn register_route(
     let password = if is_guest {
         None
     } else {
-        body.password.clone()
-    }
-    .unwrap_or_default();
+        body.password.as_deref()
+    };
 
     // Create user
-    db.users.create(&user_id, &password)?;
+    db.users.create(&user_id, password)?;
+
+    let displayname = format!("{} ⚡️", user_id.localpart());
+
+    db.users
+        .set_displayname(&user_id, Some(displayname.clone()))?;
 
     // Initial data
     db.account_data.update(
@@ -177,7 +205,7 @@ pub async fn register_route(
         EventType::PushRules,
         &ruma::events::push_rules::PushRulesEvent {
             content: ruma::events::push_rules::PushRulesEventContent {
-                global: crate::push_rules::default_pushrules(&user_id),
+                global: push::Ruleset::server_default(&user_id),
             },
         },
         &db.globals,
@@ -212,15 +240,25 @@ pub async fn register_route(
     )?;
 
     // If this is the first user on this server, create the admins room
-    if db.users.count() == 1 {
+    if db.users.count()? == 1 {
         // Create a user for the server
         let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name())
             .expect("@conduit:server_name is valid");
 
-        db.users.create(&conduit_user, "")?;
+        db.users.create(&conduit_user, None)?;
 
         let room_id = RoomId::new(db.globals.server_name());
 
+        let mutex_state = Arc::clone(
+            db.globals
+                .roomid_mutex_state
+                .write()
+                .unwrap()
+                .entry(room_id.clone())
+                .or_default(),
+        );
+        let state_lock = mutex_state.lock().await;
+
         let mut content = ruma::events::room::create::CreateEventContent::new(conduit_user.clone());
         content.federate = true;
         content.predecessor = None;
@@ -237,10 +275,8 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         // 2. Make conduit bot join
@@ -253,6 +289,7 @@ pub async fn register_route(
                     avatar_url: None,
                     is_direct: None,
                     third_party_invite: None,
+                    blurhash: None,
                 })
                 .expect("event is valid, we just created it"),
                 unsigned: None,
@@ -261,10 +298,8 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         // 3. Power levels
@@ -277,18 +312,8 @@ pub async fn register_route(
                 event_type: EventType::RoomPowerLevels,
                 content: serde_json::to_value(
                     ruma::events::room::power_levels::PowerLevelsEventContent {
-                        ban: 50.into(),
-                        events: BTreeMap::new(),
-                        events_default: 0.into(),
-                        invite: 50.into(),
-                        kick: 50.into(),
-                        redact: 50.into(),
-                        state_default: 50.into(),
                         users,
-                        users_default: 0.into(),
-                        notifications: ruma::events::room::power_levels::NotificationPowerLevels {
-                            room: 50.into(),
-                        },
+                        ..Default::default()
                     },
                 )
                 .expect("event is valid, we just created it"),
@@ -298,10 +323,8 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         // 4.1 Join Rules
@@ -318,10 +341,8 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         // 4.2 History Visibility
@@ -340,10 +361,8 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         // 4.3 Guest Access
@@ -360,32 +379,27 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         // 6. Events implied by name and topic
+        let room_name =
+            Box::<RoomName>::try_from(format!("{} Admin Room", db.globals.server_name()))
+                .expect("Room name is valid");
         db.rooms.build_and_append_pdu(
             PduBuilder {
                 event_type: EventType::RoomName,
-                content: serde_json::to_value(
-                    name::NameEventContent::new("Admin Room".to_owned()).map_err(|_| {
-                        Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.")
-                    })?,
-                )
-                .expect("event is valid, we just created it"),
+                content: serde_json::to_value(name::NameEventContent::new(Some(room_name)))
+                    .expect("event is valid, we just created it"),
                 unsigned: None,
                 state_key: Some("".to_owned()),
                 redacts: None,
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         db.rooms.build_and_append_pdu(
@@ -401,10 +415,8 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         // Room alias
@@ -426,10 +438,8 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?;
@@ -444,6 +454,7 @@ pub async fn register_route(
                     avatar_url: None,
                     is_direct: None,
                     third_party_invite: None,
+                    blurhash: None,
                 })
                 .expect("event is valid, we just created it"),
                 unsigned: None,
@@ -452,20 +463,19 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
         db.rooms.build_and_append_pdu(
             PduBuilder {
                 event_type: EventType::RoomMember,
                 content: serde_json::to_value(member::MemberEventContent {
                     membership: member::MembershipState::Join,
-                    displayname: None,
+                    displayname: Some(displayname),
                     avatar_url: None,
                     is_direct: None,
                     third_party_invite: None,
+                    blurhash: None,
                 })
                 .expect("event is valid, we just created it"),
                 unsigned: None,
@@ -474,26 +484,17 @@ pub async fn register_route(
             },
             &user_id,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
 
         // Send welcome message
         db.rooms.build_and_append_pdu(
             PduBuilder {
                 event_type: EventType::RoomMessage,
-                content: serde_json::to_value(message::MessageEventContent::Text(
-                    message::TextMessageEventContent {
-                        body: "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing `/join #conduit:matrix.org`. **Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.** Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
-                        formatted: Some(message::FormattedBody {
-                            format: message::MessageFormat::Html,
-                            body: "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing <code>/join #conduit:matrix.org</code>. <strong>Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.</strong> Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
-                        }),
-                        relates_to: None,
-                        new_content: None,
-                    },
+                content: serde_json::to_value(message::MessageEventContent::text_html(
+                        "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing `/join #conduit:matrix.org`. **Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.** Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
+                        "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing <code>/join #conduit:matrix.org</code>. <strong>Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.</strong> Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
                 ))
                 .expect("event is valid, we just created it"),
                 unsigned: None,
@@ -502,16 +503,14 @@ pub async fn register_route(
             },
             &conduit_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
     }
 
     info!("{} registered on this server", user_id);
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(register::Response {
         access_token: Some(token),
@@ -532,8 +531,9 @@ pub async fn register_route(
     feature = "conduit_bin",
     post("/_matrix/client/r0/account/password", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn change_password_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<change_password::Request<'_>>,
 ) -> ConduitResult<change_password::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -562,29 +562,33 @@ pub async fn change_password_route(
             return Err(Error::Uiaa(uiaainfo));
         }
     // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
         uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(&sender_user, &sender_device, &uiaainfo, &json)?;
         return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
     }
 
-    db.users.set_password(&sender_user, &body.new_password)?;
+    db.users
+        .set_password(&sender_user, Some(&body.new_password))?;
 
-    // TODO: Read logout_devices field when it's available and respect that, currently not supported in Ruma
-    // See: https://github.com/ruma/ruma/issues/107
-    // Logout all devices except the current one
-    for id in db
-        .users
-        .all_device_ids(&sender_user)
-        .filter_map(|id| id.ok())
-        .filter(|id| id != sender_device)
-    {
-        db.users.remove_device(&sender_user, &id)?;
+    if body.logout_devices {
+        // Logout all devices except the current one
+        for id in db
+            .users
+            .all_device_ids(&sender_user)
+            .filter_map(|id| id.ok())
+            .filter(|id| id != sender_device)
+        {
+            db.users.remove_device(&sender_user, &id)?;
+        }
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(change_password::Response.into())
+    Ok(change_password::Response {}.into())
 }
 
 /// # `GET _matrix/client/r0/account/whoami`
@@ -596,6 +600,7 @@ pub async fn change_password_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/account/whoami", data = "<body>")
 )]
+#[tracing::instrument(skip(body))]
 pub async fn whoami_route(body: Ruma<whoami::Request>) -> ConduitResult<whoami::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
     Ok(whoami::Response {
@@ -616,8 +621,9 @@ pub async fn whoami_route(body: Ruma<whoami::Request>) -> ConduitResult<whoami::
     feature = "conduit_bin",
     post("/_matrix/client/r0/account/deactivate", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn deactivate_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<deactivate::Request<'_>>,
 ) -> ConduitResult<deactivate::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -646,18 +652,27 @@ pub async fn deactivate_route(
             return Err(Error::Uiaa(uiaainfo));
         }
     // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
         uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(&sender_user, &sender_device, &uiaainfo, &json)?;
         return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
     }
 
     // Leave all joined rooms and reject all invitations
-    for room_id in db
+    let all_rooms = db
         .rooms
         .rooms_joined(&sender_user)
-        .chain(db.rooms.rooms_invited(&sender_user))
-    {
+        .chain(
+            db.rooms
+                .rooms_invited(&sender_user)
+                .map(|t| t.map(|(r, _)| r)),
+        )
+        .collect::<Vec<_>>();
+
+    for room_id in all_rooms {
         let room_id = room_id?;
         let event = member::MemberEventContent {
             membership: member::MembershipState::Leave,
@@ -665,8 +680,19 @@ pub async fn deactivate_route(
             avatar_url: None,
             is_direct: None,
             third_party_invite: None,
+            blurhash: None,
         };
 
+        let mutex_state = Arc::clone(
+            db.globals
+                .roomid_mutex_state
+                .write()
+                .unwrap()
+                .entry(room_id.clone())
+                .or_default(),
+        );
+        let state_lock = mutex_state.lock().await;
+
         db.rooms.build_and_append_pdu(
             PduBuilder {
                 event_type: EventType::RoomMember,
@@ -677,10 +703,8 @@ pub async fn deactivate_route(
             },
             &sender_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
     }
 
@@ -689,10 +713,25 @@ pub async fn deactivate_route(
 
     info!("{} deactivated their account", sender_user);
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(deactivate::Response {
         id_server_unbind_result: ThirdPartyIdRemovalStatus::NoSupport,
     }
     .into())
 }
+
+/// # `GET _matrix/client/r0/account/3pid`
+///
+/// Get a list of third party identifiers associated with this account.
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/account/3pid", data = "<body>")
+)]
+pub async fn third_party_route(
+    body: Ruma<get_contacts::Request>,
+) -> ConduitResult<get_contacts::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    Ok(get_contacts::Response::new(Vec::new()).into())
+}

src/client_server/alias.rs

@@ -1,7 +1,8 @@
-use super::State;
-use crate::{server_server, ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Ruma};
+use regex::Regex;
 use ruma::{
     api::{
+        appservice,
         client::{
             error::ErrorKind,
             r0::alias::{create_alias, delete_alias, get_alias},
@@ -18,8 +19,9 @@ use rocket::{delete, get, put};
     feature = "conduit_bin",
     put("/_matrix/client/r0/directory/room/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn create_alias_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<create_alias::Request<'_>>,
 ) -> ConduitResult<create_alias::Response> {
     if db.rooms.id_from_alias(&body.room_alias)?.is_some() {
@@ -29,7 +31,7 @@ pub async fn create_alias_route(
     db.rooms
         .set_alias(&body.room_alias, Some(&body.room_id), &db.globals)?;
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(create_alias::Response::new().into())
 }
@@ -38,13 +40,14 @@ pub async fn create_alias_route(
     feature = "conduit_bin",
     delete("/_matrix/client/r0/directory/room/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn delete_alias_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<delete_alias::Request<'_>>,
 ) -> ConduitResult<delete_alias::Response> {
     db.rooms.set_alias(&body.room_alias, None, &db.globals)?;
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(delete_alias::Response::new().into())
 }
@@ -53,8 +56,9 @@ pub async fn delete_alias_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/directory/room/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_alias_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_alias::Request<'_>>,
 ) -> ConduitResult<get_alias::Response> {
     get_alias_helper(&db, &body.room_alias).await
@@ -65,23 +69,65 @@ pub async fn get_alias_helper(
     room_alias: &RoomAliasId,
 ) -> ConduitResult<get_alias::Response> {
     if room_alias.server_name() != db.globals.server_name() {
-        let response = server_server::send_request(
-            &db.globals,
-            room_alias.server_name().to_owned(),
-            federation::query::get_room_information::v1::Request { room_alias },
-        )
-        .await?;
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                room_alias.server_name(),
+                federation::query::get_room_information::v1::Request { room_alias },
+            )
+            .await?;
 
         return Ok(get_alias::Response::new(response.room_id, response.servers).into());
     }
 
-    let room_id = db
-        .rooms
-        .id_from_alias(&room_alias)?
-        .ok_or(Error::BadRequest(
-            ErrorKind::NotFound,
-            "Room with alias not found.",
-        ))?;
+    let mut room_id = None;
+    match db.rooms.id_from_alias(&room_alias)? {
+        Some(r) => room_id = Some(r),
+        None => {
+            for (_id, registration) in db.appservice.all()? {
+                let aliases = registration
+                    .get("namespaces")
+                    .and_then(|ns| ns.get("aliases"))
+                    .and_then(|aliases| aliases.as_sequence())
+                    .map_or_else(Vec::new, |aliases| {
+                        aliases
+                            .iter()
+                            .filter_map(|aliases| Regex::new(aliases.get("regex")?.as_str()?).ok())
+                            .collect::<Vec<_>>()
+                    });
+
+                if aliases
+                    .iter()
+                    .any(|aliases| aliases.is_match(room_alias.as_str()))
+                    && db
+                        .sending
+                        .send_appservice_request(
+                            &db.globals,
+                            registration,
+                            appservice::query::query_room_alias::v1::Request { room_alias },
+                        )
+                        .await
+                        .is_ok()
+                {
+                    room_id = Some(db.rooms.id_from_alias(&room_alias)?.ok_or_else(|| {
+                        Error::bad_config("Appservice lied to us. Room does not exist.")
+                    })?);
+                    break;
+                }
+            }
+        }
+    };
+
+    let room_id = match room_id {
+        Some(room_id) => room_id,
+        None => {
+            return Err(Error::BadRequest(
+                ErrorKind::NotFound,
+                "Room with alias not found.",
+            ))
+        }
+    };
 
     Ok(get_alias::Response::new(room_id, vec![db.globals.server_name().to_owned()]).into())
 }

src/client_server/backup.rs

@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::api::client::{
     error::ErrorKind,
     r0::backup::{
@@ -17,8 +16,9 @@ use rocket::{delete, get, post, put};
     feature = "conduit_bin",
     post("/_matrix/client/unstable/room_keys/version", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn create_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<create_backup::Request>,
 ) -> ConduitResult<create_backup::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -26,7 +26,7 @@ pub async fn create_backup_route(
         .key_backups
         .create_backup(&sender_user, &body.algorithm, &db.globals)?;
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(create_backup::Response { version }.into())
 }
@@ -35,25 +35,27 @@ pub async fn create_backup_route(
     feature = "conduit_bin",
     put("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn update_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<update_backup::Request<'_>>,
 ) -> ConduitResult<update_backup::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
     db.key_backups
         .update_backup(&sender_user, &body.version, &body.algorithm, &db.globals)?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(update_backup::Response.into())
+    Ok(update_backup::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/version", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_latest_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_latest_backup::Request>,
 ) -> ConduitResult<get_latest_backup::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -79,8 +81,9 @@ pub async fn get_latest_backup_route(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_backup::Request<'_>>,
 ) -> ConduitResult<get_backup::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -105,17 +108,18 @@ pub async fn get_backup_route(
     feature = "conduit_bin",
     delete("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn delete_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<delete_backup::Request<'_>>,
 ) -> ConduitResult<delete_backup::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     db.key_backups.delete_backup(&sender_user, &body.version)?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(delete_backup::Response.into())
+    Ok(delete_backup::Response {}.into())
 }
 
 /// Add the received backup keys to the database.
@@ -123,8 +127,9 @@ pub async fn delete_backup_route(
     feature = "conduit_bin",
     put("/_matrix/client/unstable/room_keys/keys", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn add_backup_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<add_backup_keys::Request<'_>>,
 ) -> ConduitResult<add_backup_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -142,7 +147,7 @@ pub async fn add_backup_keys_route(
         }
     }
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(add_backup_keys::Response {
         count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -156,8 +161,9 @@ pub async fn add_backup_keys_route(
     feature = "conduit_bin",
     put("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn add_backup_key_sessions_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<add_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<add_backup_key_sessions::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -173,7 +179,7 @@ pub async fn add_backup_key_sessions_route(
         )?
     }
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(add_backup_key_sessions::Response {
         count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -187,8 +193,9 @@ pub async fn add_backup_key_sessions_route(
     feature = "conduit_bin",
     put("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn add_backup_key_session_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<add_backup_key_session::Request<'_>>,
 ) -> ConduitResult<add_backup_key_session::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -202,7 +209,7 @@ pub async fn add_backup_key_session_route(
         &db.globals,
     )?;
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(add_backup_key_session::Response {
         count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -215,8 +222,9 @@ pub async fn add_backup_key_session_route(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/keys", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_backup_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_backup_keys::Request<'_>>,
 ) -> ConduitResult<get_backup_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -230,15 +238,16 @@ pub async fn get_backup_keys_route(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_backup_key_sessions_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<get_backup_key_sessions::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     let sessions = db
         .key_backups
-        .get_room(&sender_user, &body.version, &body.room_id);
+        .get_room(&sender_user, &body.version, &body.room_id)?;
 
     Ok(get_backup_key_sessions::Response { sessions }.into())
 }
@@ -247,8 +256,9 @@ pub async fn get_backup_key_sessions_route(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_backup_key_session_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_backup_key_session::Request<'_>>,
 ) -> ConduitResult<get_backup_key_session::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -256,12 +266,10 @@ pub async fn get_backup_key_session_route(
     let key_data = db
         .key_backups
         .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)?
-        .ok_or_else(|| {
-            Error::BadRequest(
-                ErrorKind::NotFound,
-                "Backup key not found for this user's session.",
-            )
-        })?;
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "Backup key not found for this user's session.",
+        ))?;
 
     Ok(get_backup_key_session::Response { key_data }.into())
 }
@@ -270,8 +278,9 @@ pub async fn get_backup_key_session_route(
     feature = "conduit_bin",
     delete("/_matrix/client/unstable/room_keys/keys", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn delete_backup_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<delete_backup_keys::Request<'_>>,
 ) -> ConduitResult<delete_backup_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -279,7 +288,7 @@ pub async fn delete_backup_keys_route(
     db.key_backups
         .delete_all_keys(&sender_user, &body.version)?;
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(delete_backup_keys::Response {
         count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -292,8 +301,9 @@ pub async fn delete_backup_keys_route(
     feature = "conduit_bin",
     delete("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn delete_backup_key_sessions_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<delete_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<delete_backup_key_sessions::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -301,7 +311,7 @@ pub async fn delete_backup_key_sessions_route(
     db.key_backups
         .delete_room_keys(&sender_user, &body.version, &body.room_id)?;
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(delete_backup_key_sessions::Response {
         count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -314,8 +324,9 @@ pub async fn delete_backup_key_sessions_route(
     feature = "conduit_bin",
     delete("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn delete_backup_key_session_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<delete_backup_key_session::Request<'_>>,
 ) -> ConduitResult<delete_backup_key_session::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -323,7 +334,7 @@ pub async fn delete_backup_key_session_route(
     db.key_backups
         .delete_room_key(&sender_user, &body.version, &body.room_id, &body.session_id)?;
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(delete_backup_key_session::Response {
         count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),

src/client_server/capabilities.rs

@@ -1,5 +1,11 @@
 use crate::ConduitResult;
-use ruma::{api::client::r0::capabilities::get_capabilities, RoomVersionId};
+use crate::Ruma;
+use ruma::{
+    api::client::r0::capabilities::{
+        get_capabilities, Capabilities, RoomVersionStability, RoomVersionsCapability,
+    },
+    RoomVersionId,
+};
 use std::collections::BTreeMap;
 
 #[cfg(feature = "conduit_bin")]
@@ -8,27 +14,23 @@ use rocket::get;
 /// # `GET /_matrix/client/r0/capabilities`
 ///
 /// Get information on this server's supported feature set and other relevent capabilities.
-#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/capabilities"))]
-pub async fn get_capabilities_route() -> ConduitResult<get_capabilities::Response> {
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/capabilities", data = "<_body>")
+)]
+#[tracing::instrument(skip(_body))]
+pub async fn get_capabilities_route(
+    _body: Ruma<get_capabilities::Request>,
+) -> ConduitResult<get_capabilities::Response> {
     let mut available = BTreeMap::new();
-    available.insert(
-        RoomVersionId::Version5,
-        get_capabilities::RoomVersionStability::Stable,
-    );
-    available.insert(
-        RoomVersionId::Version6,
-        get_capabilities::RoomVersionStability::Stable,
-    );
+    available.insert(RoomVersionId::Version5, RoomVersionStability::Stable);
+    available.insert(RoomVersionId::Version6, RoomVersionStability::Stable);
 
-    Ok(get_capabilities::Response {
-        capabilities: get_capabilities::Capabilities {
-            change_password: get_capabilities::ChangePasswordCapability::default(), // enabled by default
-            room_versions: get_capabilities::RoomVersionsCapability {
-                default: RoomVersionId::Version6,
-                available,
-            },
-            custom_capabilities: BTreeMap::new(),
-        },
-    }
-    .into())
+    let mut capabilities = Capabilities::new();
+    capabilities.room_versions = RoomVersionsCapability {
+        default: RoomVersionId::Version6,
+        available,
+    };
+
+    Ok(get_capabilities::Response { capabilities }.into())
 }

src/client_server/config.rs

@@ -1,13 +1,17 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::config::{get_global_account_data, set_global_account_data},
+        r0::config::{
+            get_global_account_data, get_room_account_data, set_global_account_data,
+            set_room_account_data,
+        },
     },
-    events::{custom::CustomEventContent, BasicEvent},
+    events::{AnyGlobalAccountDataEventContent, AnyRoomAccountDataEventContent},
     serde::Raw,
 };
+use serde::Deserialize;
+use serde_json::{json, value::RawValue as RawJsonValue};
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
@@ -16,13 +20,14 @@ use rocket::{get, put};
     feature = "conduit_bin",
     put("/_matrix/client/r0/user/<_>/account_data/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn set_global_account_data_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<set_global_account_data::Request<'_>>,
 ) -> ConduitResult<set_global_account_data::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    let content = serde_json::from_str::<serde_json::Value>(body.data.get())
+    let data = serde_json::from_str::<serde_json::Value>(body.data.get())
         .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?;
 
     let event_type = body.event_type.to_string();
@@ -31,36 +36,114 @@ pub async fn set_global_account_data_route(
         None,
         sender_user,
         event_type.clone().into(),
-        &BasicEvent {
-            content: CustomEventContent {
-                event_type,
-                json: content,
-            },
-        },
+        &json!({
+            "type": event_type,
+            "content": data,
+        }),
         &db.globals,
     )?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(set_global_account_data::Response.into())
+    Ok(set_global_account_data::Response {}.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    put(
+        "/_matrix/client/r0/user/<_>/rooms/<_>/account_data/<_>",
+        data = "<body>"
+    )
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn set_room_account_data_route(
+    db: DatabaseGuard,
+    body: Ruma<set_room_account_data::Request<'_>>,
+) -> ConduitResult<set_room_account_data::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let data = serde_json::from_str::<serde_json::Value>(body.data.get())
+        .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?;
+
+    let event_type = body.event_type.to_string();
+
+    db.account_data.update(
+        Some(&body.room_id),
+        sender_user,
+        event_type.clone().into(),
+        &json!({
+            "type": event_type,
+            "content": data,
+        }),
+        &db.globals,
+    )?;
+
+    db.flush()?;
+
+    Ok(set_room_account_data::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/user/<_>/account_data/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_global_account_data_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_global_account_data::Request<'_>>,
 ) -> ConduitResult<get_global_account_data::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    let data = db
+    let event = db
         .account_data
-        .get::<Raw<ruma::events::AnyBasicEvent>>(None, sender_user, body.event_type.clone().into())?
+        .get::<Box<RawJsonValue>>(None, sender_user, body.event_type.clone().into())?
         .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
+    db.flush()?;
 
-    db.flush().await?;
+    let account_data = serde_json::from_str::<ExtractGlobalEventContent>(event.get())
+        .map_err(|_| Error::bad_database("Invalid account data event in db."))?
+        .content;
 
-    Ok(get_global_account_data::Response { account_data: data }.into())
+    Ok(get_global_account_data::Response { account_data }.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    get(
+        "/_matrix/client/r0/user/<_>/rooms/<_>/account_data/<_>",
+        data = "<body>"
+    )
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_room_account_data_route(
+    db: DatabaseGuard,
+    body: Ruma<get_room_account_data::Request<'_>>,
+) -> ConduitResult<get_room_account_data::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let event = db
+        .account_data
+        .get::<Box<RawJsonValue>>(
+            Some(&body.room_id),
+            sender_user,
+            body.event_type.clone().into(),
+        )?
+        .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
+    db.flush()?;
+
+    let account_data = serde_json::from_str::<ExtractRoomEventContent>(event.get())
+        .map_err(|_| Error::bad_database("Invalid account data event in db."))?
+        .content;
+
+    Ok(get_room_account_data::Response { account_data }.into())
+}
+
+#[derive(Deserialize)]
+struct ExtractRoomEventContent {
+    content: Raw<AnyRoomAccountDataEventContent>,
+}
+
+#[derive(Deserialize)]
+struct ExtractGlobalEventContent {
+    content: Raw<AnyGlobalAccountDataEventContent>,
 }

src/client_server/context.rs

@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::api::client::{error::ErrorKind, r0::context::get_context};
 use std::convert::TryFrom;
 
@@ -10,8 +9,9 @@ use rocket::get;
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/context/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_context_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_context::Request<'_>>,
 ) -> ConduitResult<get_context::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -23,20 +23,25 @@ pub async fn get_context_route(
         ));
     }
 
+    let base_pdu_id = db
+        .rooms
+        .get_pdu_id(&body.event_id)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "Base event id not found.",
+        ))?;
+
+    let base_token = db.rooms.pdu_count(&base_pdu_id)?;
+
     let base_event = db
         .rooms
-        .get_pdu(&body.event_id)?
+        .get_pdu_from_id(&base_pdu_id)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "Base event not found.",
         ))?
         .to_room_event();
 
-    let base_token = db
-        .rooms
-        .get_pdu_count(&body.event_id)?
-        .expect("event still exists");
-
     let events_before = db
         .rooms
         .pdus_until(&sender_user, &body.room_id, base_token)

src/client_server/device.rs

@@ -1,5 +1,4 @@
-use super::State;
-use crate::{utils, ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, utils, ConduitResult, Error, Ruma};
 use ruma::api::client::{
     error::ErrorKind,
     r0::{
@@ -16,8 +15,9 @@ use rocket::{delete, get, post, put};
     feature = "conduit_bin",
     get("/_matrix/client/r0/devices", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_devices_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_devices::Request>,
 ) -> ConduitResult<get_devices::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -35,8 +35,9 @@ pub async fn get_devices_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/devices/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_device_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_device::Request<'_>>,
 ) -> ConduitResult<get_device::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -53,8 +54,9 @@ pub async fn get_device_route(
     feature = "conduit_bin",
     put("/_matrix/client/r0/devices/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn update_device_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<update_device::Request<'_>>,
 ) -> ConduitResult<update_device::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -69,17 +71,18 @@ pub async fn update_device_route(
     db.users
         .update_device_metadata(&sender_user, &body.device_id, &device)?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(update_device::Response.into())
+    Ok(update_device::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/r0/devices/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn delete_device_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<delete_device::Request<'_>>,
 ) -> ConduitResult<delete_device::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -109,25 +112,29 @@ pub async fn delete_device_route(
             return Err(Error::Uiaa(uiaainfo));
         }
     // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
         uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(&sender_user, &sender_device, &uiaainfo, &json)?;
         return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
     }
 
     db.users.remove_device(&sender_user, &body.device_id)?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(delete_device::Response.into())
+    Ok(delete_device::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/delete_devices", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn delete_devices_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<delete_devices::Request<'_>>,
 ) -> ConduitResult<delete_devices::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -157,17 +164,20 @@ pub async fn delete_devices_route(
             return Err(Error::Uiaa(uiaainfo));
         }
     // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
         uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(&sender_user, &sender_device, &uiaainfo, &json)?;
         return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
     }
 
     for device_id in &body.devices {
         db.users.remove_device(&sender_user, &device_id)?
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(delete_devices::Response.into())
+    Ok(delete_devices::Response {}.into())
 }

src/client_server/directory.rs

@@ -1,6 +1,6 @@
-use super::State;
-use crate::{server_server, ConduitResult, Database, Error, Result, Ruma};
-use log::info;
+use std::convert::TryInto;
+
+use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Result, Ruma};
 use ruma::{
     api::{
         client::{
@@ -21,8 +21,9 @@ use ruma::{
         EventType,
     },
     serde::Raw,
-    ServerName,
+    ServerName, UInt,
 };
+use tracing::{info, warn};
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post, put};
@@ -31,8 +32,9 @@ use rocket::{get, post, put};
     feature = "conduit_bin",
     post("/_matrix/client/r0/publicRooms", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_public_rooms_filtered_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_public_rooms_filtered::Request<'_>>,
 ) -> ConduitResult<get_public_rooms_filtered::Response> {
     get_public_rooms_filtered_helper(
@@ -50,8 +52,9 @@ pub async fn get_public_rooms_filtered_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/publicRooms", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_public_rooms_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_public_rooms::Request<'_>>,
 ) -> ConduitResult<get_public_rooms::Response> {
     let response = get_public_rooms_filtered_helper(
@@ -78,37 +81,39 @@ pub async fn get_public_rooms_route(
     feature = "conduit_bin",
     put("/_matrix/client/r0/directory/list/room/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn set_room_visibility_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<set_room_visibility::Request<'_>>,
 ) -> ConduitResult<set_room_visibility::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     match &body.visibility {
-        room::Visibility::_Custom(_s) => {
-            return Err(Error::BadRequest(
-                ErrorKind::InvalidParam,
-                "Room visibility type is not supported.",
-            ));
-        }
         room::Visibility::Public => {
             db.rooms.set_public(&body.room_id, true)?;
             info!("{} made {} public", sender_user, body.room_id);
         }
         room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?,
+        _ => {
+            return Err(Error::BadRequest(
+                ErrorKind::InvalidParam,
+                "Room visibility type is not supported.",
+            ));
+        }
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(set_room_visibility::Response.into())
+    Ok(set_room_visibility::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/directory/list/room/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_room_visibility_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_room_visibility::Request<'_>>,
 ) -> ConduitResult<get_room_visibility::Response> {
     Ok(get_room_visibility::Response {
@@ -124,28 +129,28 @@ pub async fn get_room_visibility_route(
 pub async fn get_public_rooms_filtered_helper(
     db: &Database,
     server: Option<&ServerName>,
-    limit: Option<js_int::UInt>,
+    limit: Option<UInt>,
     since: Option<&str>,
     filter: &IncomingFilter,
     _network: &IncomingRoomNetwork,
 ) -> ConduitResult<get_public_rooms_filtered::Response> {
-    if let Some(other_server) = server
-        .clone()
-        .filter(|server| *server != db.globals.server_name().as_str())
+    if let Some(other_server) = server.filter(|server| *server != db.globals.server_name().as_str())
     {
-        let response = server_server::send_request(
-            &db.globals,
-            other_server.to_owned(),
-            federation::directory::get_public_rooms_filtered::v1::Request {
-                limit,
-                since: since.as_deref(),
-                filter: Filter {
-                    generic_search_term: filter.generic_search_term.as_deref(),
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                other_server,
+                federation::directory::get_public_rooms_filtered::v1::Request {
+                    limit,
+                    since: since.as_deref(),
+                    filter: Filter {
+                        generic_search_term: filter.generic_search_term.as_deref(),
+                    },
+                    room_network: RoomNetwork::Matrix,
                 },
-                room_network: RoomNetwork::Matrix,
-            },
-        )
-        .await?;
+            )
+            .await?;
 
         return Ok(get_public_rooms_filtered::Response {
             chunk: response
@@ -154,15 +159,12 @@ pub async fn get_public_rooms_filtered_helper(
                 .map(|c| {
                     // Convert ruma::api::federation::directory::get_public_rooms::v1::PublicRoomsChunk
                     // to ruma::api::client::r0::directory::PublicRoomsChunk
-                    Ok::<_, Error>(
-                        serde_json::from_str(
-                            &serde_json::to_string(&c)
-                                .expect("PublicRoomsChunk::to_string always works"),
-                        )
-                        .expect("federation and client-server PublicRoomsChunk are the same type"),
+                    serde_json::from_str(
+                        &serde_json::to_string(&c)
+                            .expect("PublicRoomsChunk::to_string always works"),
                     )
+                    .expect("federation and client-server PublicRoomsChunk are the same type")
                 })
-                .filter_map(|r| r.ok())
                 .collect(),
             prev_batch: response.prev_batch,
             next_batch: response.next_batch,
@@ -203,13 +205,11 @@ pub async fn get_public_rooms_filtered_helper(
             .map(|room_id| {
                 let room_id = room_id?;
 
-                // TODO: Do not load full state?
-                let state = db.rooms.room_state_full(&room_id)?;
-
                 let chunk = PublicRoomsChunk {
                     aliases: Vec::new(),
-                    canonical_alias: state
-                        .get(&(EventType::RoomCanonicalAlias, "".to_owned()))
+                    canonical_alias: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")?
                         .map_or(Ok::<_, Error>(None), |s| {
                             Ok(serde_json::from_value::<
                                 Raw<canonical_alias::CanonicalAliasEventContent>,
@@ -221,9 +221,10 @@ pub async fn get_public_rooms_filtered_helper(
                             })?
                             .alias)
                         })?,
-                    name: state.get(&(EventType::RoomName, "".to_owned())).map_or(
-                        Ok::<_, Error>(None),
-                        |s| {
+                    name: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomName, "")?
+                        .map_or(Ok::<_, Error>(None), |s| {
                             Ok(serde_json::from_value::<Raw<name::NameEventContent>>(
                                 s.content.clone(),
                             )
@@ -232,15 +233,22 @@ pub async fn get_public_rooms_filtered_helper(
                             .map_err(|_| {
                                 Error::bad_database("Invalid room name event in database.")
                             })?
-                            .name()
-                            .map(|n| n.to_owned()))
-                        },
-                    )?,
-                    num_joined_members: (db.rooms.room_members(&room_id).count() as u32).into(),
-                    room_id,
-                    topic: state.get(&(EventType::RoomTopic, "".to_owned())).map_or(
-                        Ok::<_, Error>(None),
-                        |s| {
+                            .name
+                            .map(|n| n.to_owned().into()))
+                        })?,
+                    num_joined_members: db
+                        .rooms
+                        .room_joined_count(&room_id)?
+                        .unwrap_or_else(|| {
+                            warn!("Room {} has no member count", room_id);
+                            0
+                        })
+                        .try_into()
+                        .expect("user count should not be that big"),
+                    topic: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomTopic, "")?
+                        .map_or(Ok::<_, Error>(None), |s| {
                             Ok(Some(
                                 serde_json::from_value::<Raw<topic::TopicEventContent>>(
                                     s.content.clone(),
@@ -252,10 +260,10 @@ pub async fn get_public_rooms_filtered_helper(
                                 })?
                                 .topic,
                             ))
-                        },
-                    )?,
-                    world_readable: state
-                        .get(&(EventType::RoomHistoryVisibility, "".to_owned()))
+                        })?,
+                    world_readable: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")?
                         .map_or(Ok::<_, Error>(false), |s| {
                             Ok(serde_json::from_value::<
                                 Raw<history_visibility::HistoryVisibilityEventContent>,
@@ -270,8 +278,9 @@ pub async fn get_public_rooms_filtered_helper(
                             .history_visibility
                                 == history_visibility::HistoryVisibility::WorldReadable)
                         })?,
-                    guest_can_join: state
-                        .get(&(EventType::RoomGuestAccess, "".to_owned()))
+                    guest_can_join: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomGuestAccess, "")?
                         .map_or(Ok::<_, Error>(false), |s| {
                             Ok(
                             serde_json::from_value::<Raw<guest_access::GuestAccessEventContent>>(
@@ -286,8 +295,9 @@ pub async fn get_public_rooms_filtered_helper(
                                 == guest_access::GuestAccess::CanJoin,
                         )
                         })?,
-                    avatar_url: state
-                        .get(&(EventType::RoomAvatar, "".to_owned()))
+                    avatar_url: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomAvatar, "")?
                         .map(|s| {
                             Ok::<_, Error>(
                                 serde_json::from_value::<Raw<avatar::AvatarEventContent>>(
@@ -304,10 +314,41 @@ pub async fn get_public_rooms_filtered_helper(
                         .transpose()?
                         // url is now an Option<String> so we must flatten
                         .flatten(),
+                    room_id,
                 };
                 Ok(chunk)
             })
             .filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms
+            .filter(|chunk| {
+                if let Some(query) = filter
+                    .generic_search_term
+                    .as_ref()
+                    .map(|q| q.to_lowercase())
+                {
+                    if let Some(name) = &chunk.name {
+                        if name.as_str().to_lowercase().contains(&query) {
+                            return true;
+                        }
+                    }
+
+                    if let Some(topic) = &chunk.topic {
+                        if topic.to_lowercase().contains(&query) {
+                            return true;
+                        }
+                    }
+
+                    if let Some(canonical_alias) = &chunk.canonical_alias {
+                        if canonical_alias.as_str().to_lowercase().contains(&query) {
+                            return true;
+                        }
+                    }
+
+                    false
+                } else {
+                    // No search term
+                    true
+                }
+            })
             // We need to collect all, so we can sort by member count
             .collect::<Vec<_>>();
 

src/client_server/filter.rs

@@ -5,6 +5,7 @@ use ruma::api::client::r0::filter::{self, create_filter, get_filter};
 use rocket::{get, post};
 
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))]
+#[tracing::instrument]
 pub async fn get_filter_route() -> ConduitResult<get_filter::Response> {
     // TODO
     Ok(get_filter::Response::new(filter::IncomingFilterDefinition {
@@ -18,6 +19,7 @@ pub async fn get_filter_route() -> ConduitResult<get_filter::Response> {
 }
 
 #[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/user/<_>/filter"))]
+#[tracing::instrument]
 pub async fn create_filter_route() -> ConduitResult<create_filter::Response> {
     // TODO
     Ok(create_filter::Response::new(utils::random_string(10)).into())

src/client_server/keys.rs

@@ -1,18 +1,23 @@
-use super::{State, SESSION_ID_LENGTH};
-use crate::{utils, ConduitResult, Database, Error, Ruma};
+use super::SESSION_ID_LENGTH;
+use crate::{database::DatabaseGuard, utils, ConduitResult, Database, Error, Result, Ruma};
 use ruma::{
-    api::client::{
-        error::ErrorKind,
-        r0::{
-            keys::{
-                claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures,
-                upload_signing_keys,
+    api::{
+        client::{
+            error::ErrorKind,
+            r0::{
+                keys::{
+                    claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures,
+                    upload_signing_keys,
+                },
+                uiaa::{AuthFlow, UiaaInfo},
             },
-            uiaa::{AuthFlow, UiaaInfo},
         },
+        federation,
     },
     encryption::UnsignedDeviceInfo,
+    DeviceId, DeviceKeyAlgorithm, UserId,
 };
+use serde_json::json;
 use std::collections::{BTreeMap, HashSet};
 
 #[cfg(feature = "conduit_bin")]
@@ -22,8 +27,9 @@ use rocket::{get, post};
     feature = "conduit_bin",
     post("/_matrix/client/r0/keys/upload", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn upload_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<upload_keys::Request>,
 ) -> ConduitResult<upload_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -58,7 +64,7 @@ pub async fn upload_keys_route(
         }
     }
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(upload_keys::Response {
         one_time_key_counts: db.users.count_one_time_keys(sender_user, sender_device)?,
@@ -70,121 +76,47 @@ pub async fn upload_keys_route(
     feature = "conduit_bin",
     post("/_matrix/client/r0/keys/query", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_keys::Request<'_>>,
 ) -> ConduitResult<get_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    let mut master_keys = BTreeMap::new();
-    let mut self_signing_keys = BTreeMap::new();
-    let mut user_signing_keys = BTreeMap::new();
-    let mut device_keys = BTreeMap::new();
+    let response = get_keys_helper(
+        Some(sender_user),
+        &body.device_keys,
+        |u| u == sender_user,
+        &db,
+    )
+    .await?;
 
-    for (user_id, device_ids) in &body.device_keys {
-        if device_ids.is_empty() {
-            let mut container = BTreeMap::new();
-            for device_id in db.users.all_device_ids(user_id) {
-                let device_id = device_id?;
-                if let Some(mut keys) = db.users.get_device_keys(user_id, &device_id)? {
-                    let metadata = db
-                        .users
-                        .get_device_metadata(user_id, &device_id)?
-                        .ok_or_else(|| {
-                            Error::bad_database("all_device_keys contained nonexistent device.")
-                        })?;
-
-                    keys.unsigned = UnsignedDeviceInfo {
-                        device_display_name: metadata.display_name,
-                    };
-
-                    container.insert(device_id, keys);
-                }
-            }
-            device_keys.insert(user_id.clone(), container);
-        } else {
-            for device_id in device_ids {
-                let mut container = BTreeMap::new();
-                if let Some(mut keys) = db.users.get_device_keys(&user_id.clone(), &device_id)? {
-                    let metadata = db.users.get_device_metadata(user_id, &device_id)?.ok_or(
-                        Error::BadRequest(
-                            ErrorKind::InvalidParam,
-                            "Tried to get keys for nonexistent device.",
-                        ),
-                    )?;
-
-                    keys.unsigned = UnsignedDeviceInfo {
-                        device_display_name: metadata.display_name,
-                    };
-
-                    container.insert(device_id.clone(), keys);
-                }
-                device_keys.insert(user_id.clone(), container);
-            }
-        }
-
-        if let Some(master_key) = db.users.get_master_key(user_id, sender_user)? {
-            master_keys.insert(user_id.clone(), master_key);
-        }
-        if let Some(self_signing_key) = db.users.get_self_signing_key(user_id, sender_user)? {
-            self_signing_keys.insert(user_id.clone(), self_signing_key);
-        }
-        if user_id == sender_user {
-            if let Some(user_signing_key) = db.users.get_user_signing_key(sender_user)? {
-                user_signing_keys.insert(user_id.clone(), user_signing_key);
-            }
-        }
-    }
-
-    Ok(get_keys::Response {
-        master_keys,
-        self_signing_keys,
-        user_signing_keys,
-        device_keys,
-        failures: BTreeMap::new(),
-    }
-    .into())
+    Ok(response.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/keys/claim", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn claim_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<claim_keys::Request>,
 ) -> ConduitResult<claim_keys::Response> {
-    let mut one_time_keys = BTreeMap::new();
-    for (user_id, map) in &body.one_time_keys {
-        let mut container = BTreeMap::new();
-        for (device_id, key_algorithm) in map {
-            if let Some(one_time_keys) =
-                db.users
-                    .take_one_time_key(user_id, device_id, key_algorithm, &db.globals)?
-            {
-                let mut c = BTreeMap::new();
-                c.insert(one_time_keys.0, one_time_keys.1);
-                container.insert(device_id.clone(), c);
-            }
-        }
-        one_time_keys.insert(user_id.clone(), container);
-    }
+    let response = claim_keys_helper(&body.one_time_keys, &db).await?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(claim_keys::Response {
-        failures: BTreeMap::new(),
-        one_time_keys,
-    }
-    .into())
+    Ok(response.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/unstable/keys/device_signing/upload", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn upload_signing_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<upload_signing_keys::Request<'_>>,
 ) -> ConduitResult<upload_signing_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -214,10 +146,13 @@ pub async fn upload_signing_keys_route(
             return Err(Error::Uiaa(uiaainfo));
         }
     // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
         uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(&sender_user, &sender_device, &uiaainfo, &json)?;
         return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
     }
 
     if let Some(master_key) = &body.master_key {
@@ -231,17 +166,18 @@ pub async fn upload_signing_keys_route(
         )?;
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(upload_signing_keys::Response.into())
+    Ok(upload_signing_keys::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/unstable/keys/signatures/upload", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn upload_signatures_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<upload_signatures::Request>,
 ) -> ConduitResult<upload_signatures::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -291,17 +227,18 @@ pub async fn upload_signatures_route(
         }
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(upload_signatures::Response.into())
+    Ok(upload_signatures::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/keys/changes", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_key_changes_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_key_changes::Request<'_>>,
 ) -> ConduitResult<get_key_changes::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -345,3 +282,176 @@ pub async fn get_key_changes_route(
     }
     .into())
 }
+
+pub async fn get_keys_helper<F: Fn(&UserId) -> bool>(
+    sender_user: Option<&UserId>,
+    device_keys_input: &BTreeMap<UserId, Vec<Box<DeviceId>>>,
+    allowed_signatures: F,
+    db: &Database,
+) -> Result<get_keys::Response> {
+    let mut master_keys = BTreeMap::new();
+    let mut self_signing_keys = BTreeMap::new();
+    let mut user_signing_keys = BTreeMap::new();
+    let mut device_keys = BTreeMap::new();
+
+    let mut get_over_federation = BTreeMap::new();
+
+    for (user_id, device_ids) in device_keys_input {
+        if user_id.server_name() != db.globals.server_name() {
+            get_over_federation
+                .entry(user_id.server_name())
+                .or_insert_with(Vec::new)
+                .push((user_id, device_ids));
+            continue;
+        }
+
+        if device_ids.is_empty() {
+            let mut container = BTreeMap::new();
+            for device_id in db.users.all_device_ids(user_id) {
+                let device_id = device_id?;
+                if let Some(mut keys) = db.users.get_device_keys(user_id, &device_id)? {
+                    let metadata = db
+                        .users
+                        .get_device_metadata(user_id, &device_id)?
+                        .ok_or_else(|| {
+                            Error::bad_database("all_device_keys contained nonexistent device.")
+                        })?;
+
+                    keys.unsigned = UnsignedDeviceInfo {
+                        device_display_name: metadata.display_name,
+                    };
+
+                    container.insert(device_id, keys);
+                }
+            }
+            device_keys.insert(user_id.clone(), container);
+        } else {
+            for device_id in device_ids {
+                let mut container = BTreeMap::new();
+                if let Some(mut keys) = db.users.get_device_keys(&user_id.clone(), &device_id)? {
+                    let metadata = db.users.get_device_metadata(user_id, &device_id)?.ok_or(
+                        Error::BadRequest(
+                            ErrorKind::InvalidParam,
+                            "Tried to get keys for nonexistent device.",
+                        ),
+                    )?;
+
+                    keys.unsigned = UnsignedDeviceInfo {
+                        device_display_name: metadata.display_name,
+                    };
+
+                    container.insert(device_id.clone(), keys);
+                }
+                device_keys.insert(user_id.clone(), container);
+            }
+        }
+
+        if let Some(master_key) = db.users.get_master_key(user_id, &allowed_signatures)? {
+            master_keys.insert(user_id.clone(), master_key);
+        }
+        if let Some(self_signing_key) = db
+            .users
+            .get_self_signing_key(user_id, &allowed_signatures)?
+        {
+            self_signing_keys.insert(user_id.clone(), self_signing_key);
+        }
+        if Some(user_id) == sender_user {
+            if let Some(user_signing_key) = db.users.get_user_signing_key(user_id)? {
+                user_signing_keys.insert(user_id.clone(), user_signing_key);
+            }
+        }
+    }
+
+    let mut failures = BTreeMap::new();
+
+    for (server, vec) in get_over_federation {
+        let mut device_keys_input_fed = BTreeMap::new();
+        for (user_id, keys) in vec {
+            device_keys_input_fed.insert(user_id.clone(), keys.clone());
+        }
+        match db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                server,
+                federation::keys::get_keys::v1::Request {
+                    device_keys: device_keys_input_fed,
+                },
+            )
+            .await
+        {
+            Ok(response) => {
+                master_keys.extend(response.master_keys);
+                self_signing_keys.extend(response.self_signing_keys);
+                device_keys.extend(response.device_keys);
+            }
+            Err(_e) => {
+                failures.insert(server.to_string(), json!({}));
+            }
+        }
+    }
+
+    Ok(get_keys::Response {
+        master_keys,
+        self_signing_keys,
+        user_signing_keys,
+        device_keys,
+        failures,
+    })
+}
+
+pub async fn claim_keys_helper(
+    one_time_keys_input: &BTreeMap<UserId, BTreeMap<Box<DeviceId>, DeviceKeyAlgorithm>>,
+    db: &Database,
+) -> Result<claim_keys::Response> {
+    let mut one_time_keys = BTreeMap::new();
+
+    let mut get_over_federation = BTreeMap::new();
+
+    for (user_id, map) in one_time_keys_input {
+        if user_id.server_name() != db.globals.server_name() {
+            get_over_federation
+                .entry(user_id.server_name())
+                .or_insert_with(Vec::new)
+                .push((user_id, map));
+        }
+
+        let mut container = BTreeMap::new();
+        for (device_id, key_algorithm) in map {
+            if let Some(one_time_keys) =
+                db.users
+                    .take_one_time_key(user_id, device_id, key_algorithm, &db.globals)?
+            {
+                let mut c = BTreeMap::new();
+                c.insert(one_time_keys.0, one_time_keys.1);
+                container.insert(device_id.clone(), c);
+            }
+        }
+        one_time_keys.insert(user_id.clone(), container);
+    }
+
+    for (server, vec) in get_over_federation {
+        let mut one_time_keys_input_fed = BTreeMap::new();
+        for (user_id, keys) in vec {
+            one_time_keys_input_fed.insert(user_id.clone(), keys.clone());
+        }
+        // Ignore failures
+        let keys = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                server,
+                federation::keys::claim_keys::v1::Request {
+                    one_time_keys: one_time_keys_input_fed,
+                },
+            )
+            .await?;
+
+        one_time_keys.extend(keys.one_time_keys);
+    }
+
+    Ok(claim_keys::Response {
+        failures: BTreeMap::new(),
+        one_time_keys,
+    })
+}

src/client_server/media.rs

@@ -1,21 +1,21 @@
-use super::State;
 use crate::{
-    database::media::FileMeta, server_server, utils, ConduitResult, Database, Error, Ruma,
+    database::media::FileMeta, database::DatabaseGuard, utils, ConduitResult, Error, Ruma,
 };
 use ruma::api::client::{
     error::ErrorKind,
     r0::media::{create_content, get_content, get_content_thumbnail, get_media_config},
 };
+use std::convert::TryInto;
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};
-use std::convert::TryInto;
 
 const MXC_LENGTH: usize = 32;
 
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/media/r0/config"))]
+#[tracing::instrument(skip(db))]
 pub async fn get_media_config_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
 ) -> ConduitResult<get_media_config::Response> {
     Ok(get_media_config::Response {
         upload_size: db.globals.max_request_size().into(),
@@ -27,8 +27,9 @@ pub async fn get_media_config_route(
     feature = "conduit_bin",
     post("/_matrix/media/r0/upload", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn create_content_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<create_content::Request<'_>>,
 ) -> ConduitResult<create_content::Response> {
     let mxc = format!(
@@ -36,58 +37,76 @@ pub async fn create_content_route(
         db.globals.server_name(),
         utils::random_string(MXC_LENGTH)
     );
-    db.media.create(
-        mxc.clone(),
-        &body.filename.as_deref(),
-        &body.content_type.as_deref(),
-        &body.file,
-    )?;
 
-    db.flush().await?;
+    db.media
+        .create(
+            mxc.clone(),
+            &db.globals,
+            &body
+                .filename
+                .as_ref()
+                .map(|filename| "inline; filename=".to_owned() + filename)
+                .as_deref(),
+            &body.content_type.as_deref(),
+            &body.file,
+        )
+        .await?;
 
-    Ok(create_content::Response { content_uri: mxc }.into())
+    db.flush()?;
+
+    Ok(create_content::Response {
+        content_uri: mxc.try_into().expect("Invalid mxc:// URI"),
+        blurhash: None,
+    }
+    .into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/media/r0/download/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_content_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_content::Request<'_>>,
 ) -> ConduitResult<get_content::Response> {
     let mxc = format!("mxc://{}/{}", body.server_name, body.media_id);
 
     if let Some(FileMeta {
-        filename,
+        content_disposition,
         content_type,
         file,
-    }) = db.media.get(&mxc)?
+    }) = db.media.get(&db.globals, &mxc).await?
     {
         Ok(get_content::Response {
             file,
             content_type,
-            content_disposition: filename,
+            content_disposition,
         }
         .into())
     } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
-        let get_content_response = server_server::send_request(
-            &db.globals,
-            body.server_name.clone(),
-            get_content::Request {
-                allow_remote: false,
-                server_name: &body.server_name,
-                media_id: &body.media_id,
-            },
-        )
-        .await?;
+        let get_content_response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                &body.server_name,
+                get_content::Request {
+                    allow_remote: false,
+                    server_name: &body.server_name,
+                    media_id: &body.media_id,
+                },
+            )
+            .await?;
 
-        db.media.create(
-            mxc,
-            &get_content_response.content_disposition.as_deref(),
-            &get_content_response.content_type.as_deref(),
-            &get_content_response.file,
-        )?;
+        db.media
+            .create(
+                mxc,
+                &db.globals,
+                &get_content_response.content_disposition.as_deref(),
+                &get_content_response.content_type.as_deref(),
+                &get_content_response.file,
+            )
+            .await?;
 
         Ok(get_content_response.into())
     } else {
@@ -99,47 +118,58 @@ pub async fn get_content_route(
     feature = "conduit_bin",
     get("/_matrix/media/r0/thumbnail/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_content_thumbnail_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_content_thumbnail::Request<'_>>,
 ) -> ConduitResult<get_content_thumbnail::Response> {
     let mxc = format!("mxc://{}/{}", body.server_name, body.media_id);
 
     if let Some(FileMeta {
         content_type, file, ..
-    }) = db.media.get_thumbnail(
-        mxc.clone(),
-        body.width
-            .try_into()
-            .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
-        body.height
-            .try_into()
-            .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
-    )? {
+    }) = db
+        .media
+        .get_thumbnail(
+            mxc.clone(),
+            &db.globals,
+            body.width
+                .try_into()
+                .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
+            body.height
+                .try_into()
+                .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
+        )
+        .await?
+    {
         Ok(get_content_thumbnail::Response { file, content_type }.into())
     } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
-        let get_thumbnail_response = server_server::send_request(
-            &db.globals,
-            body.server_name.clone(),
-            get_content_thumbnail::Request {
-                allow_remote: false,
-                height: body.height,
-                width: body.width,
-                method: body.method,
-                server_name: &body.server_name,
-                media_id: &body.media_id,
-            },
-        )
-        .await?;
+        let get_thumbnail_response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                &body.server_name,
+                get_content_thumbnail::Request {
+                    allow_remote: false,
+                    height: body.height,
+                    width: body.width,
+                    method: body.method.clone(),
+                    server_name: &body.server_name,
+                    media_id: &body.media_id,
+                },
+            )
+            .await?;
 
-        db.media.upload_thumbnail(
-            mxc,
-            &None,
-            &get_thumbnail_response.content_type,
-            body.width.try_into().expect("all UInts are valid u32s"),
-            body.height.try_into().expect("all UInts are valid u32s"),
-            &get_thumbnail_response.file,
-        )?;
+        db.media
+            .upload_thumbnail(
+                mxc,
+                &db.globals,
+                &None,
+                &get_thumbnail_response.content_type,
+                body.width.try_into().expect("all UInts are valid u32s"),
+                body.height.try_into().expect("all UInts are valid u32s"),
+                &get_thumbnail_response.file,
+            )
+            .await?;
 
         Ok(get_thumbnail_response.into())
     } else {

src/client_server/message.rs

@@ -1,14 +1,17 @@
-use super::State;
-use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma};
 use ruma::{
     api::client::{
         error::ErrorKind,
         r0::message::{get_message_events, send_message_event},
     },
-    events::EventContent,
+    events::EventType,
     EventId,
 };
-use std::convert::{TryFrom, TryInto};
+use std::{
+    collections::BTreeMap,
+    convert::{TryFrom, TryInto},
+    sync::Arc,
+};
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
@@ -17,12 +20,23 @@ use rocket::{get, put};
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/send/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn send_message_event_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<send_message_event::Request<'_>>,
 ) -> ConduitResult<send_message_event::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-    let sender_device = body.sender_device.as_ref().expect("user is authenticated");
+    let sender_device = body.sender_device.as_deref();
+
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(body.room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
 
     // Check if this is a new transaction id
     if let Some(response) =
@@ -46,29 +60,22 @@ pub async fn send_message_event_route(
         return Ok(send_message_event::Response { event_id }.into());
     }
 
-    let mut unsigned = serde_json::Map::new();
+    let mut unsigned = BTreeMap::new();
     unsigned.insert("transaction_id".to_owned(), body.txn_id.clone().into());
 
     let event_id = db.rooms.build_and_append_pdu(
         PduBuilder {
-            event_type: body.content.event_type().into(),
-            content: serde_json::from_str(
-                body.json_body
-                    .as_ref()
-                    .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?
-                    .get(),
-            )
-            .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?,
+            event_type: EventType::from(&body.event_type),
+            content: serde_json::from_str(body.body.body.json().get())
+                .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?,
             unsigned: Some(unsigned),
             state_key: None,
             redacts: None,
         },
         &sender_user,
         &body.room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     db.transaction_ids.add_txnid(
@@ -78,7 +85,9 @@ pub async fn send_message_event_route(
         event_id.as_bytes(),
     )?;
 
-    db.flush().await?;
+    drop(state_lock);
+
+    db.flush()?;
 
     Ok(send_message_event::Response::new(event_id).into())
 }
@@ -87,8 +96,9 @@ pub async fn send_message_event_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/messages", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_message_events_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_message_events::Request<'_>>,
 ) -> ConduitResult<get_message_events::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");

src/client_server/mod.rs

@@ -64,17 +64,16 @@ pub use voip::*;
 use super::State;
 #[cfg(feature = "conduit_bin")]
 use {
-    crate::ConduitResult,
-    rocket::{options, State},
-    ruma::api::client::r0::to_device::send_event_to_device,
+    crate::ConduitResult, rocket::options, ruma::api::client::r0::to_device::send_event_to_device,
 };
 
-const DEVICE_ID_LENGTH: usize = 10;
-const TOKEN_LENGTH: usize = 256;
-const SESSION_ID_LENGTH: usize = 256;
+pub const DEVICE_ID_LENGTH: usize = 10;
+pub const TOKEN_LENGTH: usize = 256;
+pub const SESSION_ID_LENGTH: usize = 256;
 
 #[cfg(feature = "conduit_bin")]
 #[options("/<_..>")]
+#[tracing::instrument]
 pub async fn options_route() -> ConduitResult<send_event_to_device::Response> {
-    Ok(send_event_to_device::Response.into())
+    Ok(send_event_to_device::Response {}.into())
 }

src/client_server/presence.rs

@@ -1,17 +1,17 @@
-use super::State;
-use crate::{utils, ConduitResult, Database, Ruma};
-use ruma::api::client::r0::presence::set_presence;
-use std::convert::TryInto;
+use crate::{database::DatabaseGuard, utils, ConduitResult, Ruma};
+use ruma::api::client::r0::presence::{get_presence, set_presence};
+use std::{convert::TryInto, time::Duration};
 
 #[cfg(feature = "conduit_bin")]
-use rocket::put;
+use rocket::{get, put};
 
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/presence/<_>/status", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn set_presence_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<set_presence::Request<'_>>,
 ) -> ConduitResult<set_presence::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -41,7 +41,52 @@ pub async fn set_presence_route(
         )?;
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(set_presence::Response.into())
+    Ok(set_presence::Response {}.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/presence/<_>/status", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_presence_route(
+    db: DatabaseGuard,
+    body: Ruma<get_presence::Request<'_>>,
+) -> ConduitResult<get_presence::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let mut presence_event = None;
+
+    for room_id in db
+        .rooms
+        .get_shared_rooms(vec![sender_user.clone(), body.user_id.clone()])?
+    {
+        let room_id = room_id?;
+
+        if let Some(presence) = db
+            .rooms
+            .edus
+            .get_last_presence_event(&sender_user, &room_id)?
+        {
+            presence_event = Some(presence);
+        }
+    }
+
+    if let Some(presence) = presence_event {
+        Ok(get_presence::Response {
+            // TODO: Should ruma just use the presenceeventcontent type here?
+            status_msg: presence.content.status_msg,
+            currently_active: presence.content.currently_active,
+            last_active_ago: presence
+                .content
+                .last_active_ago
+                .map(|millis| Duration::from_millis(millis.into())),
+            presence: presence.content.presence,
+        }
+        .into())
+    } else {
+        todo!();
+    }
 }

src/client_server/profile.rs

@@ -1,26 +1,29 @@
-use super::State;
-use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma};
 use ruma::{
-    api::client::{
-        error::ErrorKind,
-        r0::profile::{
-            get_avatar_url, get_display_name, get_profile, set_avatar_url, set_display_name,
+    api::{
+        client::{
+            error::ErrorKind,
+            r0::profile::{
+                get_avatar_url, get_display_name, get_profile, set_avatar_url, set_display_name,
+            },
         },
+        federation::{self, query::get_profile_information::v1::ProfileField},
     },
     events::EventType,
     serde::Raw,
 };
+use std::{convert::TryInto, sync::Arc};
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
-use std::convert::TryInto;
 
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/profile/<_>/displayname", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn set_displayname_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<set_display_name::Request<'_>>,
 ) -> ConduitResult<set_display_name::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -29,45 +32,60 @@ pub async fn set_displayname_route(
         .set_displayname(&sender_user, body.displayname.clone())?;
 
     // Send a new membership event and presence update into all joined rooms
-    for room_id in db.rooms.rooms_joined(&sender_user) {
-        let room_id = room_id?;
-        db.rooms.build_and_append_pdu(
-            PduBuilder {
-                event_type: EventType::RoomMember,
-                content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
-                    displayname: body.displayname.clone(),
-                    ..serde_json::from_value::<Raw<_>>(
-                        db.rooms
-                            .room_state_get(
-                                &room_id,
-                                &EventType::RoomMember,
-                                &sender_user.to_string(),
-                            )?
-                            .ok_or_else(|| {
-                                Error::bad_database(
-                                    "Tried to send displayname update for user not in the room.",
-                                )
-                            })?
-                            .1
-                            .content
-                            .clone(),
-                    )
-                    .expect("from_value::<Raw<..>> can never fail")
-                    .deserialize()
-                    .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
-                })
-                .expect("event is valid, we just created it"),
-                unsigned: None,
-                state_key: Some(sender_user.to_string()),
-                redacts: None,
-            },
-            &sender_user,
-            &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
-        )?;
+    let all_rooms_joined = db.rooms.rooms_joined(&sender_user).collect::<Vec<_>>();
+
+    for (pdu_builder, room_id) in all_rooms_joined
+        .into_iter()
+        .filter_map(|r| r.ok())
+        .map(|room_id| {
+            Ok::<_, Error>((
+                PduBuilder {
+                    event_type: EventType::RoomMember,
+                    content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
+                        displayname: body.displayname.clone(),
+                        ..serde_json::from_value::<Raw<_>>(
+                            db.rooms
+                                .room_state_get(
+                                    &room_id,
+                                    &EventType::RoomMember,
+                                    &sender_user.to_string(),
+                                )?
+                                .ok_or_else(|| {
+                                    Error::bad_database(
+                                        "Tried to send displayname update for user not in the \
+                                         room.",
+                                    )
+                                })?
+                                .content
+                                .clone(),
+                        )
+                        .expect("from_value::<Raw<..>> can never fail")
+                        .deserialize()
+                        .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
+                    })
+                    .expect("event is valid, we just created it"),
+                    unsigned: None,
+                    state_key: Some(sender_user.to_string()),
+                    redacts: None,
+                },
+                room_id,
+            ))
+        })
+        .filter_map(|r| r.ok())
+    {
+        let mutex_state = Arc::clone(
+            db.globals
+                .roomid_mutex_state
+                .write()
+                .unwrap()
+                .entry(room_id.clone())
+                .or_default(),
+        );
+        let state_lock = mutex_state.lock().await;
+
+        let _ =
+            db.rooms
+                .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock);
 
         // Presence update
         db.rooms.edus.update_presence(
@@ -92,19 +110,39 @@ pub async fn set_displayname_route(
         )?;
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(set_display_name::Response.into())
+    Ok(set_display_name::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/profile/<_>/displayname", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_displayname_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_display_name::Request<'_>>,
 ) -> ConduitResult<get_display_name::Response> {
+    if body.user_id.server_name() != db.globals.server_name() {
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                body.user_id.server_name(),
+                federation::query::get_profile_information::v1::Request {
+                    user_id: &body.user_id,
+                    field: Some(&ProfileField::DisplayName),
+                },
+            )
+            .await?;
+
+        return Ok(get_display_name::Response {
+            displayname: response.displayname,
+        }
+        .into());
+    }
+
     Ok(get_display_name::Response {
         displayname: db.users.displayname(&body.user_id)?,
     }
@@ -115,8 +153,9 @@ pub async fn get_displayname_route(
     feature = "conduit_bin",
     put("/_matrix/client/r0/profile/<_>/avatar_url", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn set_avatar_url_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<set_avatar_url::Request<'_>>,
 ) -> ConduitResult<set_avatar_url::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -124,46 +163,63 @@ pub async fn set_avatar_url_route(
     db.users
         .set_avatar_url(&sender_user, body.avatar_url.clone())?;
 
+    db.users.set_blurhash(&sender_user, body.blurhash.clone())?;
+
     // Send a new membership event and presence update into all joined rooms
-    for room_id in db.rooms.rooms_joined(&sender_user) {
-        let room_id = room_id?;
-        db.rooms.build_and_append_pdu(
-            PduBuilder {
-                event_type: EventType::RoomMember,
-                content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
-                    avatar_url: body.avatar_url.clone(),
-                    ..serde_json::from_value::<Raw<_>>(
-                        db.rooms
-                            .room_state_get(
-                                &room_id,
-                                &EventType::RoomMember,
-                                &sender_user.to_string(),
-                            )?
-                            .ok_or_else(|| {
-                                Error::bad_database(
-                                    "Tried to send avatar url update for user not in the room.",
-                                )
-                            })?
-                            .1
-                            .content
-                            .clone(),
-                    )
-                    .expect("from_value::<Raw<..>> can never fail")
-                    .deserialize()
-                    .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
-                })
-                .expect("event is valid, we just created it"),
-                unsigned: None,
-                state_key: Some(sender_user.to_string()),
-                redacts: None,
-            },
-            &sender_user,
-            &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
-        )?;
+    let all_joined_rooms = db.rooms.rooms_joined(&sender_user).collect::<Vec<_>>();
+
+    for (pdu_builder, room_id) in all_joined_rooms
+        .into_iter()
+        .filter_map(|r| r.ok())
+        .map(|room_id| {
+            Ok::<_, Error>((
+                PduBuilder {
+                    event_type: EventType::RoomMember,
+                    content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
+                        avatar_url: body.avatar_url.clone(),
+                        ..serde_json::from_value::<Raw<_>>(
+                            db.rooms
+                                .room_state_get(
+                                    &room_id,
+                                    &EventType::RoomMember,
+                                    &sender_user.to_string(),
+                                )?
+                                .ok_or_else(|| {
+                                    Error::bad_database(
+                                        "Tried to send displayname update for user not in the \
+                                         room.",
+                                    )
+                                })?
+                                .content
+                                .clone(),
+                        )
+                        .expect("from_value::<Raw<..>> can never fail")
+                        .deserialize()
+                        .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
+                    })
+                    .expect("event is valid, we just created it"),
+                    unsigned: None,
+                    state_key: Some(sender_user.to_string()),
+                    redacts: None,
+                },
+                room_id,
+            ))
+        })
+        .filter_map(|r| r.ok())
+    {
+        let mutex_state = Arc::clone(
+            db.globals
+                .roomid_mutex_state
+                .write()
+                .unwrap()
+                .entry(room_id.clone())
+                .or_default(),
+        );
+        let state_lock = mutex_state.lock().await;
+
+        let _ =
+            db.rooms
+                .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock);
 
         // Presence update
         db.rooms.edus.update_presence(
@@ -188,21 +244,43 @@ pub async fn set_avatar_url_route(
         )?;
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(set_avatar_url::Response.into())
+    Ok(set_avatar_url::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/profile/<_>/avatar_url", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_avatar_url_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_avatar_url::Request<'_>>,
 ) -> ConduitResult<get_avatar_url::Response> {
+    if body.user_id.server_name() != db.globals.server_name() {
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                body.user_id.server_name(),
+                federation::query::get_profile_information::v1::Request {
+                    user_id: &body.user_id,
+                    field: Some(&ProfileField::AvatarUrl),
+                },
+            )
+            .await?;
+
+        return Ok(get_avatar_url::Response {
+            avatar_url: response.avatar_url,
+            blurhash: response.blurhash,
+        }
+        .into());
+    }
+
     Ok(get_avatar_url::Response {
         avatar_url: db.users.avatar_url(&body.user_id)?,
+        blurhash: db.users.blurhash(&body.user_id)?,
     }
     .into())
 }
@@ -211,10 +289,32 @@ pub async fn get_avatar_url_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/profile/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_profile_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_profile::Request<'_>>,
 ) -> ConduitResult<get_profile::Response> {
+    if body.user_id.server_name() != db.globals.server_name() {
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                body.user_id.server_name(),
+                federation::query::get_profile_information::v1::Request {
+                    user_id: &body.user_id,
+                    field: None,
+                },
+            )
+            .await?;
+
+        return Ok(get_profile::Response {
+            displayname: response.displayname,
+            avatar_url: response.avatar_url,
+            blurhash: response.blurhash,
+        }
+        .into());
+    }
+
     if !db.users.exists(&body.user_id)? {
         // Return 404 if this user doesn't exist
         return Err(Error::BadRequest(
@@ -225,6 +325,7 @@ pub async fn get_profile_route(
 
     Ok(get_profile::Response {
         avatar_url: db.users.avatar_url(&body.user_id)?,
+        blurhash: db.users.blurhash(&body.user_id)?,
         displayname: db.users.displayname(&body.user_id)?,
     }
     .into())

src/client_server/push.rs

@@ -1,30 +1,34 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
-use log::warn;
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::push::{get_pushers, get_pushrules_all, set_pushrule, set_pushrule_enabled},
+        r0::push::{
+            delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled,
+            get_pushrules_all, set_pusher, set_pushrule, set_pushrule_actions,
+            set_pushrule_enabled, RuleKind,
+        },
     },
-    events::EventType,
+    events::{push_rules, EventType},
+    push::{ConditionalPushRuleInit, PatternedPushRuleInit, SimplePushRuleInit},
 };
 
 #[cfg(feature = "conduit_bin")]
-use rocket::{get, post, put};
+use rocket::{delete, get, post, put};
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/pushrules", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_pushrules_all_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_pushrules_all::Request>,
 ) -> ConduitResult<get_pushrules_all::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     let event = db
         .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "PushRules event not found.",
@@ -36,51 +40,537 @@ pub async fn get_pushrules_all_route(
     .into())
 }
 
-#[cfg_attr(feature = "conduit_bin", put(
-    "/_matrix/client/r0/pushrules/<_>/<_>/<_>",
-    //data = "<body>"
-))]
-pub async fn set_pushrule_route(
-    db: State<'_, Database>,
-    //body: Ruma<set_pushrule::Request>,
-) -> ConduitResult<set_pushrule::Response> {
-    // TODO
-    warn!("TODO: set_pushrule_route");
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_pushrule_route(
+    db: DatabaseGuard,
+    body: Ruma<get_pushrule::Request<'_>>,
+) -> ConduitResult<get_pushrule::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    db.flush().await?;
+    let event = db
+        .account_data
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
 
-    Ok(set_pushrule::Response.into())
+    let global = event.content.global;
+    let rule = match body.kind {
+        RuleKind::Override => global
+            .override_
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
+        RuleKind::Underride => global
+            .underride
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
+        RuleKind::Sender => global
+            .sender
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
+        RuleKind::Room => global
+            .room
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
+        RuleKind::Content => global
+            .content
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
+        _ => None,
+    };
+
+    if let Some(rule) = rule {
+        Ok(get_pushrule::Response { rule }.into())
+    } else {
+        Err(Error::BadRequest(
+            ErrorKind::NotFound,
+            "Push rule not found.",
+        ))
+    }
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
-    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled")
+    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<req>")
 )]
+#[tracing::instrument(skip(db, req))]
+pub async fn set_pushrule_route(
+    db: DatabaseGuard,
+    req: Ruma<set_pushrule::Request<'_>>,
+) -> ConduitResult<set_pushrule::Response> {
+    let sender_user = req.sender_user.as_ref().expect("user is authenticated");
+    let body = req.body;
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    match body.kind {
+        RuleKind::Override => {
+            global.override_.replace(
+                ConditionalPushRuleInit {
+                    actions: body.actions,
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id,
+                    conditions: body.conditions,
+                }
+                .into(),
+            );
+        }
+        RuleKind::Underride => {
+            global.underride.replace(
+                ConditionalPushRuleInit {
+                    actions: body.actions,
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id,
+                    conditions: body.conditions,
+                }
+                .into(),
+            );
+        }
+        RuleKind::Sender => {
+            global.sender.replace(
+                SimplePushRuleInit {
+                    actions: body.actions,
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id,
+                }
+                .into(),
+            );
+        }
+        RuleKind::Room => {
+            global.room.replace(
+                SimplePushRuleInit {
+                    actions: body.actions,
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id,
+                }
+                .into(),
+            );
+        }
+        RuleKind::Content => {
+            global.content.replace(
+                PatternedPushRuleInit {
+                    actions: body.actions,
+                    default: false,
+                    enabled: true,
+                    rule_id: body.rule_id,
+                    pattern: body.pattern.unwrap_or_default(),
+                }
+                .into(),
+            );
+        }
+        _ => {}
+    }
+
+    db.account_data.update(
+        None,
+        &sender_user,
+        EventType::PushRules,
+        &event,
+        &db.globals,
+    )?;
+
+    db.flush()?;
+
+    Ok(set_pushrule::Response {}.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_pushrule_actions_route(
+    db: DatabaseGuard,
+    body: Ruma<get_pushrule_actions::Request<'_>>,
+) -> ConduitResult<get_pushrule_actions::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    let actions = match body.kind {
+        RuleKind::Override => global
+            .override_
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
+        RuleKind::Underride => global
+            .underride
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
+        RuleKind::Sender => global
+            .sender
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
+        RuleKind::Room => global
+            .room
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
+        RuleKind::Content => global
+            .content
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
+        _ => None,
+    };
+
+    db.flush()?;
+
+    Ok(get_pushrule_actions::Response {
+        actions: actions.unwrap_or_default(),
+    }
+    .into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn set_pushrule_actions_route(
+    db: DatabaseGuard,
+    body: Ruma<set_pushrule_actions::Request<'_>>,
+) -> ConduitResult<set_pushrule_actions::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    match body.kind {
+        RuleKind::Override => {
+            if let Some(mut rule) = global.override_.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.override_.replace(rule);
+            }
+        }
+        RuleKind::Underride => {
+            if let Some(mut rule) = global.underride.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.underride.replace(rule);
+            }
+        }
+        RuleKind::Sender => {
+            if let Some(mut rule) = global.sender.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.sender.replace(rule);
+            }
+        }
+        RuleKind::Room => {
+            if let Some(mut rule) = global.room.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.room.replace(rule);
+            }
+        }
+        RuleKind::Content => {
+            if let Some(mut rule) = global.content.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.content.replace(rule);
+            }
+        }
+        _ => {}
+    };
+
+    db.account_data.update(
+        None,
+        &sender_user,
+        EventType::PushRules,
+        &event,
+        &db.globals,
+    )?;
+
+    db.flush()?;
+
+    Ok(set_pushrule_actions::Response {}.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_pushrule_enabled_route(
+    db: DatabaseGuard,
+    body: Ruma<get_pushrule_enabled::Request<'_>>,
+) -> ConduitResult<get_pushrule_enabled::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    let enabled = match body.kind {
+        RuleKind::Override => global
+            .override_
+            .iter()
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
+        RuleKind::Underride => global
+            .underride
+            .iter()
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
+        RuleKind::Sender => global
+            .sender
+            .iter()
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
+        RuleKind::Room => global
+            .room
+            .iter()
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
+        RuleKind::Content => global
+            .content
+            .iter()
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
+        _ => false,
+    };
+
+    db.flush()?;
+
+    Ok(get_pushrule_enabled::Response { enabled }.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
 pub async fn set_pushrule_enabled_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
+    body: Ruma<set_pushrule_enabled::Request<'_>>,
 ) -> ConduitResult<set_pushrule_enabled::Response> {
-    // TODO
-    warn!("TODO: set_pushrule_enabled_route");
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    db.flush().await?;
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
 
-    Ok(set_pushrule_enabled::Response.into())
+    let mut event = db
+        .account_data
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    match body.kind {
+        RuleKind::Override => {
+            if let Some(mut rule) = global.override_.get(body.rule_id.as_str()).cloned() {
+                global.override_.remove(&rule);
+                rule.enabled = body.enabled;
+                global.override_.insert(rule);
+            }
+        }
+        RuleKind::Underride => {
+            if let Some(mut rule) = global.underride.get(body.rule_id.as_str()).cloned() {
+                global.underride.remove(&rule);
+                rule.enabled = body.enabled;
+                global.underride.insert(rule);
+            }
+        }
+        RuleKind::Sender => {
+            if let Some(mut rule) = global.sender.get(body.rule_id.as_str()).cloned() {
+                global.sender.remove(&rule);
+                rule.enabled = body.enabled;
+                global.sender.insert(rule);
+            }
+        }
+        RuleKind::Room => {
+            if let Some(mut rule) = global.room.get(body.rule_id.as_str()).cloned() {
+                global.room.remove(&rule);
+                rule.enabled = body.enabled;
+                global.room.insert(rule);
+            }
+        }
+        RuleKind::Content => {
+            if let Some(mut rule) = global.content.get(body.rule_id.as_str()).cloned() {
+                global.content.remove(&rule);
+                rule.enabled = body.enabled;
+                global.content.insert(rule);
+            }
+        }
+        _ => {}
+    }
+
+    db.account_data.update(
+        None,
+        &sender_user,
+        EventType::PushRules,
+        &event,
+        &db.globals,
+    )?;
+
+    db.flush()?;
+
+    Ok(set_pushrule_enabled::Response {}.into())
 }
 
-#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/pushers"))]
-pub async fn get_pushers_route() -> ConduitResult<get_pushers::Response> {
+#[cfg_attr(
+    feature = "conduit_bin",
+    delete("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn delete_pushrule_route(
+    db: DatabaseGuard,
+    body: Ruma<delete_pushrule::Request<'_>>,
+) -> ConduitResult<delete_pushrule::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if body.scope != "global" {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Scopes other than 'global' are not supported.",
+        ));
+    }
+
+    let mut event = db
+        .account_data
+        .get::<push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "PushRules event not found.",
+        ))?;
+
+    let global = &mut event.content.global;
+    match body.kind {
+        RuleKind::Override => {
+            if let Some(rule) = global.override_.get(body.rule_id.as_str()).cloned() {
+                global.override_.remove(&rule);
+            }
+        }
+        RuleKind::Underride => {
+            if let Some(rule) = global.underride.get(body.rule_id.as_str()).cloned() {
+                global.underride.remove(&rule);
+            }
+        }
+        RuleKind::Sender => {
+            if let Some(rule) = global.sender.get(body.rule_id.as_str()).cloned() {
+                global.sender.remove(&rule);
+            }
+        }
+        RuleKind::Room => {
+            if let Some(rule) = global.room.get(body.rule_id.as_str()).cloned() {
+                global.room.remove(&rule);
+            }
+        }
+        RuleKind::Content => {
+            if let Some(rule) = global.content.get(body.rule_id.as_str()).cloned() {
+                global.content.remove(&rule);
+            }
+        }
+        _ => {}
+    }
+
+    db.account_data.update(
+        None,
+        &sender_user,
+        EventType::PushRules,
+        &event,
+        &db.globals,
+    )?;
+
+    db.flush()?;
+
+    Ok(delete_pushrule::Response {}.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/pushers", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_pushers_route(
+    db: DatabaseGuard,
+    body: Ruma<get_pushers::Request>,
+) -> ConduitResult<get_pushers::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
     Ok(get_pushers::Response {
-        pushers: Vec::new(),
+        pushers: db.pusher.get_pushers(sender_user)?,
     }
     .into())
 }
 
-#[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/pushers/set"))]
-pub async fn set_pushers_route(db: State<'_, Database>) -> ConduitResult<get_pushers::Response> {
-    db.flush().await?;
+#[cfg_attr(
+    feature = "conduit_bin",
+    post("/_matrix/client/r0/pushers/set", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn set_pushers_route(
+    db: DatabaseGuard,
+    body: Ruma<set_pusher::Request>,
+) -> ConduitResult<set_pusher::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+    let pusher = body.pusher.clone();
 
-    Ok(get_pushers::Response {
-        pushers: Vec::new(),
-    }
-    .into())
+    db.pusher.set_pusher(sender_user, pusher)?;
+
+    db.flush()?;
+
+    Ok(set_pusher::Response::default().into())
 }

src/client_server/read_marker.rs

@@ -1,20 +1,25 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::{
-    api::client::{error::ErrorKind, r0::read_marker::set_read_marker},
-    events::{AnyEphemeralRoomEvent, AnyEvent, EventType},
+    api::client::{
+        error::ErrorKind,
+        r0::{read_marker::set_read_marker, receipt::create_receipt},
+    },
+    events::{AnyEphemeralRoomEvent, EventType},
+    receipt::ReceiptType,
+    MilliSecondsSinceUnixEpoch,
 };
+use std::collections::BTreeMap;
 
 #[cfg(feature = "conduit_bin")]
 use rocket::post;
-use std::{collections::BTreeMap, time::SystemTime};
 
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/read_markers", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn set_read_marker_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<set_read_marker::Request<'_>>,
 ) -> ConduitResult<set_read_marker::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -23,7 +28,6 @@ pub async fn set_read_marker_route(
         content: ruma::events::fully_read::FullyReadEventContent {
             event_id: body.fully_read.clone(),
         },
-        room_id: body.room_id.clone(),
     };
     db.account_data.update(
         Some(&body.room_id),
@@ -43,36 +47,88 @@ pub async fn set_read_marker_route(
             ))?,
             &db.globals,
         )?;
+        db.rooms
+            .reset_notification_counts(&sender_user, &body.room_id)?;
 
         let mut user_receipts = BTreeMap::new();
         user_receipts.insert(
             sender_user.clone(),
             ruma::events::receipt::Receipt {
-                ts: Some(SystemTime::now()),
+                ts: Some(MilliSecondsSinceUnixEpoch::now()),
             },
         );
+
+        let mut receipts = BTreeMap::new();
+        receipts.insert(ReceiptType::Read, user_receipts);
+
         let mut receipt_content = BTreeMap::new();
-        receipt_content.insert(
-            event.to_owned(),
-            ruma::events::receipt::Receipts {
-                read: Some(user_receipts),
-            },
-        );
+        receipt_content.insert(event.to_owned(), receipts);
 
         db.rooms.edus.readreceipt_update(
             &sender_user,
             &body.room_id,
-            AnyEvent::Ephemeral(AnyEphemeralRoomEvent::Receipt(
-                ruma::events::receipt::ReceiptEvent {
-                    content: ruma::events::receipt::ReceiptEventContent(receipt_content),
-                    room_id: body.room_id.clone(),
-                },
-            )),
+            AnyEphemeralRoomEvent::Receipt(ruma::events::receipt::ReceiptEvent {
+                content: ruma::events::receipt::ReceiptEventContent(receipt_content),
+                room_id: body.room_id.clone(),
+            }),
             &db.globals,
         )?;
     }
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(set_read_marker::Response.into())
+    Ok(set_read_marker::Response {}.into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    post("/_matrix/client/r0/rooms/<_>/receipt/<_>/<_>", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn create_receipt_route(
+    db: DatabaseGuard,
+    body: Ruma<create_receipt::Request<'_>>,
+) -> ConduitResult<create_receipt::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    db.rooms.edus.private_read_set(
+        &body.room_id,
+        &sender_user,
+        db.rooms
+            .get_pdu_count(&body.event_id)?
+            .ok_or(Error::BadRequest(
+                ErrorKind::InvalidParam,
+                "Event does not exist.",
+            ))?,
+        &db.globals,
+    )?;
+    db.rooms
+        .reset_notification_counts(&sender_user, &body.room_id)?;
+
+    let mut user_receipts = BTreeMap::new();
+    user_receipts.insert(
+        sender_user.clone(),
+        ruma::events::receipt::Receipt {
+            ts: Some(MilliSecondsSinceUnixEpoch::now()),
+        },
+    );
+    let mut receipts = BTreeMap::new();
+    receipts.insert(ReceiptType::Read, user_receipts);
+
+    let mut receipt_content = BTreeMap::new();
+    receipt_content.insert(body.event_id.to_owned(), receipts);
+
+    db.rooms.edus.readreceipt_update(
+        &sender_user,
+        &body.room_id,
+        AnyEphemeralRoomEvent::Receipt(ruma::events::receipt::ReceiptEvent {
+            content: ruma::events::receipt::ReceiptEventContent(receipt_content),
+            room_id: body.room_id.clone(),
+        }),
+        &db.globals,
+    )?;
+
+    db.flush()?;
+
+    Ok(create_receipt::Response {}.into())
 }

src/client_server/redact.rs

@@ -1,5 +1,6 @@
-use super::State;
-use crate::{pdu::PduBuilder, ConduitResult, Database, Ruma};
+use std::sync::Arc;
+
+use crate::{database::DatabaseGuard, pdu::PduBuilder, ConduitResult, Ruma};
 use ruma::{
     api::client::r0::redact::redact_event,
     events::{room::redaction, EventType},
@@ -12,12 +13,23 @@ use rocket::put;
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/redact/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn redact_event_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<redact_event::Request<'_>>,
 ) -> ConduitResult<redact_event::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(body.room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
     let event_id = db.rooms.build_and_append_pdu(
         PduBuilder {
             event_type: EventType::RoomRedaction,
@@ -31,13 +43,13 @@ pub async fn redact_event_route(
         },
         &sender_user,
         &body.room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
-    db.flush().await?;
+    drop(state_lock);
+
+    db.flush()?;
 
     Ok(redact_event::Response { event_id }.into())
 }

src/client_server/room.rs

@@ -1,10 +1,11 @@
-use super::State;
-use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma};
-use log::info;
+use crate::{
+    client_server::invite_helper, database::DatabaseGuard, pdu::PduBuilder, ConduitResult, Error,
+    Ruma,
+};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::room::{self, create_room, get_room_event, upgrade_room},
+        r0::room::{self, aliases, create_room, get_room_event, upgrade_room},
     },
     events::{
         room::{guest_access, history_visibility, join_rules, member, name, topic},
@@ -13,7 +14,8 @@ use ruma::{
     serde::Raw,
     RoomAliasId, RoomId, RoomVersionId,
 };
-use std::{cmp::max, collections::BTreeMap, convert::TryFrom};
+use std::{cmp::max, collections::BTreeMap, convert::TryFrom, sync::Arc};
+use tracing::{info, warn};
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};
@@ -22,14 +24,25 @@ use rocket::{get, post};
     feature = "conduit_bin",
     post("/_matrix/client/r0/createRoom", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn create_room_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<create_room::Request<'_>>,
 ) -> ConduitResult<create_room::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     let room_id = RoomId::new(db.globals.server_name());
 
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
     let alias = body
         .room_alias_name
         .as_ref()
@@ -52,7 +65,19 @@ pub async fn create_room_route(
     let mut content = ruma::events::room::create::CreateEventContent::new(sender_user.clone());
     content.federate = body.creation_content.federate;
     content.predecessor = body.creation_content.predecessor.clone();
-    content.room_version = RoomVersionId::Version6;
+    content.room_version = match body.room_version.clone() {
+        Some(room_version) => {
+            if room_version == RoomVersionId::Version5 || room_version == RoomVersionId::Version6 {
+                room_version
+            } else {
+                return Err(Error::BadRequest(
+                    ErrorKind::UnsupportedRoomVersion,
+                    "This server does not support that room version.",
+                ));
+            }
+        }
+        None => RoomVersionId::Version6,
+    };
 
     // 1. The room create event
     db.rooms.build_and_append_pdu(
@@ -65,10 +90,8 @@ pub async fn create_room_route(
         },
         &sender_user,
         &room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     // 2. Let the room creator join
@@ -81,6 +104,7 @@ pub async fn create_room_route(
                 avatar_url: db.users.avatar_url(&sender_user)?,
                 is_direct: Some(body.is_direct),
                 third_party_invite: None,
+                blurhash: db.users.blurhash(&sender_user)?,
             })
             .expect("event is valid, we just created it"),
             unsigned: None,
@@ -89,40 +113,51 @@ pub async fn create_room_route(
         },
         &sender_user,
         &room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     // 3. Power levels
+
+    // Figure out preset. We need it for preset specific events
+    let preset = body
+        .preset
+        .clone()
+        .unwrap_or_else(|| match &body.visibility {
+            room::Visibility::Private => create_room::RoomPreset::PrivateChat,
+            room::Visibility::Public => create_room::RoomPreset::PublicChat,
+            _ => create_room::RoomPreset::PrivateChat, // Room visibility should not be custom
+        });
+
     let mut users = BTreeMap::new();
     users.insert(sender_user.clone(), 100.into());
-    for invite_ in &body.invite {
-        users.insert(invite_.clone(), 100.into());
+
+    if preset == create_room::RoomPreset::TrustedPrivateChat {
+        for invite_ in &body.invite {
+            users.insert(invite_.clone(), 100.into());
+        }
     }
 
-    let power_levels_content = if let Some(power_levels) = &body.power_level_content_override {
-        serde_json::from_str(power_levels.json().get()).map_err(|_| {
-            Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.")
-        })?
-    } else {
+    let mut power_levels_content =
         serde_json::to_value(ruma::events::room::power_levels::PowerLevelsEventContent {
-            ban: 50.into(),
-            events: BTreeMap::new(),
-            events_default: 0.into(),
-            invite: 50.into(),
-            kick: 50.into(),
-            redact: 50.into(),
-            state_default: 50.into(),
             users,
-            users_default: 0.into(),
-            notifications: ruma::events::room::power_levels::NotificationPowerLevels {
-                room: 50.into(),
-            },
+            ..Default::default()
         })
-        .expect("event is valid, we just created it")
-    };
+        .expect("event is valid, we just created it");
+
+    if let Some(power_level_content_override) = &body.power_level_content_override {
+        let json = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(
+            power_level_content_override.json().get(),
+        )
+        .map_err(|_| {
+            Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.")
+        })?;
+
+        for (key, value) in json {
+            power_levels_content[key] = value;
+        }
+    }
+
     db.rooms.build_and_append_pdu(
         PduBuilder {
             event_type: EventType::RoomPowerLevels,
@@ -133,24 +168,12 @@ pub async fn create_room_route(
         },
         &sender_user,
         &room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     // 4. Events set by preset
 
-    // Figure out preset. We need it for preset specific events
-    let preset = body
-        .preset
-        .clone()
-        .unwrap_or_else(|| match &body.visibility {
-            room::Visibility::Private => create_room::RoomPreset::PrivateChat,
-            room::Visibility::Public => create_room::RoomPreset::PublicChat,
-            room::Visibility::_Custom(s) => create_room::RoomPreset::_Custom(s.into()),
-        });
-
     // 4.1 Join Rules
     db.rooms.build_and_append_pdu(
         PduBuilder {
@@ -172,10 +195,8 @@ pub async fn create_room_route(
         },
         &sender_user,
         &room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     // 4.2 History Visibility
@@ -192,10 +213,8 @@ pub async fn create_room_route(
         },
         &sender_user,
         &room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     // 4.3 Guest Access
@@ -220,33 +239,24 @@ pub async fn create_room_route(
         },
         &sender_user,
         &room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     // 5. Events listed in initial_state
     for event in &body.initial_state {
-        let pdu_builder = serde_json::from_str::<PduBuilder>(
-            &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"),
-        )
-        .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?;
+        let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| {
+            warn!("Invalid initial state event: {:?}", e);
+            Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.")
+        })?);
 
         // Silently skip encryption events if they are not allowed
-        if pdu_builder.event_type == EventType::RoomEncryption && db.globals.encryption_disabled() {
+        if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() {
             continue;
         }
 
-        db.rooms.build_and_append_pdu(
-            pdu_builder,
-            &sender_user,
-            &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
-        )?;
+        db.rooms
+            .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock)?;
     }
 
     // 6. Events implied by name and topic
@@ -254,22 +264,16 @@ pub async fn create_room_route(
         db.rooms.build_and_append_pdu(
             PduBuilder {
                 event_type: EventType::RoomName,
-                content: serde_json::to_value(
-                    name::NameEventContent::new(name.clone()).map_err(|_| {
-                        Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.")
-                    })?,
-                )
-                .expect("event is valid, we just created it"),
+                content: serde_json::to_value(name::NameEventContent::new(Some(name.clone())))
+                    .expect("event is valid, we just created it"),
                 unsigned: None,
                 state_key: Some("".to_owned()),
                 redacts: None,
             },
             &sender_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
     }
 
@@ -287,37 +291,15 @@ pub async fn create_room_route(
             },
             &sender_user,
             &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
     }
 
     // 7. Events implied by invite (and TODO: invite_3pid)
-    for user in &body.invite {
-        db.rooms.build_and_append_pdu(
-            PduBuilder {
-                event_type: EventType::RoomMember,
-                content: serde_json::to_value(member::MemberEventContent {
-                    membership: member::MembershipState::Invite,
-                    displayname: db.users.displayname(&user)?,
-                    avatar_url: db.users.avatar_url(&user)?,
-                    is_direct: Some(body.is_direct),
-                    third_party_invite: None,
-                })
-                .expect("event is valid, we just created it"),
-                unsigned: None,
-                state_key: Some(user.to_string()),
-                redacts: None,
-            },
-            &sender_user,
-            &room_id,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
-        )?;
+    drop(state_lock);
+    for user_id in &body.invite {
+        let _ = invite_helper(sender_user, user_id, &room_id, &db, body.is_direct).await;
     }
 
     // Homeserver specific stuff
@@ -331,7 +313,7 @@ pub async fn create_room_route(
 
     info!("{} created a room", sender_user);
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(create_room::Response::new(room_id).into())
 }
@@ -340,8 +322,9 @@ pub async fn create_room_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/event/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_room_event_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_room_event::Request<'_>>,
 ) -> ConduitResult<get_room_event::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -365,12 +348,40 @@ pub async fn get_room_event_route(
 
 #[cfg_attr(
     feature = "conduit_bin",
-    post("/_matrix/client/r0/rooms/<_room_id>/upgrade", data = "<body>")
+    get("/_matrix/client/r0/rooms/<_>/aliases", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_room_aliases_route(
+    db: DatabaseGuard,
+    body: Ruma<aliases::Request<'_>>,
+) -> ConduitResult<aliases::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if !db.rooms.is_joined(sender_user, &body.room_id)? {
+        return Err(Error::BadRequest(
+            ErrorKind::Forbidden,
+            "You don't have permission to view this room.",
+        ));
+    }
+
+    Ok(aliases::Response {
+        aliases: db
+            .rooms
+            .room_aliases(&body.room_id)
+            .filter_map(|a| a.ok())
+            .collect(),
+    }
+    .into())
+}
+
+#[cfg_attr(
+    feature = "conduit_bin",
+    post("/_matrix/client/r0/rooms/<_>/upgrade", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
 pub async fn upgrade_room_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<upgrade_room::Request<'_>>,
-    _room_id: String,
 ) -> ConduitResult<upgrade_room::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
@@ -387,6 +398,16 @@ pub async fn upgrade_room_route(
     // Create a replacement room
     let replacement_room = RoomId::new(db.globals.server_name());
 
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(body.room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
     // Send a m.room.tombstone event to the old room to indicate that it is not intended to be used any further
     // Fail if the sender does not have the required permissions
     let tombstone_event_id = db.rooms.build_and_append_pdu(
@@ -403,19 +424,29 @@ pub async fn upgrade_room_route(
         },
         sender_user,
         &body.room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
+    // Change lock to replacement room
+    drop(state_lock);
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(replacement_room.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
     // Get the old room federations status
     let federate = serde_json::from_value::<Raw<ruma::events::room::create::CreateEventContent>>(
         db.rooms
             .room_state_get(&body.room_id, &EventType::RoomCreate, "")?
             .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
-            .1
-            .content,
+            .content
+            .clone(),
     )
     .expect("Raw::from_value always works")
     .deserialize()
@@ -446,10 +477,8 @@ pub async fn upgrade_room_route(
         },
         sender_user,
         &replacement_room,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     // Join the new room
@@ -462,6 +491,7 @@ pub async fn upgrade_room_route(
                 avatar_url: db.users.avatar_url(&sender_user)?,
                 is_direct: None,
                 third_party_invite: None,
+                blurhash: db.users.blurhash(&sender_user)?,
             })
             .expect("event is valid, we just created it"),
             unsigned: None,
@@ -470,10 +500,8 @@ pub async fn upgrade_room_route(
         },
         sender_user,
         &replacement_room,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     // Recommended transferable state events list from the specs
@@ -492,7 +520,7 @@ pub async fn upgrade_room_route(
     // Replicate transferable state events to the new room
     for event_type in transferable_state_events {
         let event_content = match db.rooms.room_state_get(&body.room_id, &event_type, "")? {
-            Some((_, v)) => v.content.clone(),
+            Some(v) => v.content.clone(),
             None => continue, // Skipping missing events.
         };
 
@@ -506,10 +534,8 @@ pub async fn upgrade_room_route(
             },
             sender_user,
             &replacement_room,
-            &db.globals,
-            &db.sending,
-            &db.admin,
-            &db.account_data,
+            &db,
+            &state_lock,
         )?;
     }
 
@@ -525,8 +551,8 @@ pub async fn upgrade_room_route(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")?
                 .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
-                .1
-                .content,
+                .content
+                .clone(),
         )
         .expect("database contains invalid PDU")
         .deserialize()
@@ -552,13 +578,13 @@ pub async fn upgrade_room_route(
         },
         sender_user,
         &body.room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
-    db.flush().await?;
+    drop(state_lock);
+
+    db.flush()?;
 
     // Return the replacement room id
     Ok(upgrade_room::Response { replacement_room }.into())

src/client_server/search.rs

@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::api::client::{error::ErrorKind, r0::search::search_events};
 
 #[cfg(feature = "conduit_bin")]
@@ -11,24 +10,40 @@ use std::collections::BTreeMap;
     feature = "conduit_bin",
     post("/_matrix/client/r0/search", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn search_events_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<search_events::Request<'_>>,
 ) -> ConduitResult<search_events::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     let search_criteria = body.search_categories.room_events.as_ref().unwrap();
-    let filter = search_criteria.filter.as_ref().unwrap();
+    let filter = search_criteria.filter.clone().unwrap_or_default();
 
-    let room_id = filter.rooms.as_ref().unwrap().first().unwrap();
+    let room_ids = filter.rooms.clone().unwrap_or_else(|| {
+        db.rooms
+            .rooms_joined(&sender_user)
+            .filter_map(|r| r.ok())
+            .collect()
+    });
 
     let limit = filter.limit.map_or(10, |l| u64::from(l) as usize);
 
-    if !db.rooms.is_joined(sender_user, &room_id)? {
-        return Err(Error::BadRequest(
-            ErrorKind::Forbidden,
-            "You don't have permission to view this room.",
-        ));
+    let mut searches = Vec::new();
+
+    for room_id in room_ids {
+        if !db.rooms.is_joined(sender_user, &room_id)? {
+            return Err(Error::BadRequest(
+                ErrorKind::Forbidden,
+                "You don't have permission to view this room.",
+            ));
+        }
+
+        let search = db
+            .rooms
+            .search_pdus(&room_id, &search_criteria.search_term)?;
+
+        searches.push(search.0.peekable());
     }
 
     let skip = match body.next_batch.as_ref().map(|s| s.parse()) {
@@ -42,12 +57,20 @@ pub async fn search_events_route(
         None => 0, // Default to the start
     };
 
-    let search = db
-        .rooms
-        .search_pdus(&room_id, &search_criteria.search_term)?;
+    let mut results = Vec::new();
+    for _ in 0..skip + limit {
+        if let Some(s) = searches
+            .iter_mut()
+            .map(|s| (s.peek().cloned(), s))
+            .max_by_key(|(peek, _)| peek.clone())
+            .and_then(|(_, i)| i.next())
+        {
+            results.push(s);
+        }
+    }
 
-    let results = search
-        .0
+    let results = results
+        .iter()
         .map(|result| {
             Ok::<_, Error>(SearchResult {
                 context: EventContextResult {
@@ -77,12 +100,16 @@ pub async fn search_events_route(
 
     Ok(search_events::Response::new(ResultCategories {
         room_events: ResultRoomEvents {
-            count: None,             // TODO? maybe not
-            groups: BTreeMap::new(), // TODO
+            count: Some((results.len() as u32).into()), // TODO: set this to none. Element shouldn't depend on it
+            groups: BTreeMap::new(),                    // TODO
             next_batch,
             results,
             state: BTreeMap::new(), // TODO
-            highlights: search.1,
+            highlights: search_criteria
+                .search_term
+                .split_terminator(|c: char| !c.is_alphanumeric())
+                .map(str::to_lowercase)
+                .collect::<Vec<_>>(),
         },
     })
     .into())

src/client_server/session.rs

@@ -1,6 +1,5 @@
-use super::{State, DEVICE_ID_LENGTH, TOKEN_LENGTH};
-use crate::{utils, ConduitResult, Database, Error, Ruma};
-use log::info;
+use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH};
+use crate::{database::DatabaseGuard, utils, ConduitResult, Error, Ruma};
 use ruma::{
     api::client::{
         error::ErrorKind,
@@ -8,6 +7,14 @@ use ruma::{
     },
     UserId,
 };
+use serde::Deserialize;
+use tracing::info;
+
+#[derive(Debug, Deserialize)]
+struct Claims {
+    sub: String,
+    exp: usize,
+}
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};
@@ -17,8 +24,14 @@ use rocket::{get, post};
 /// Get the homeserver's supported login types. One of these should be used as the `type` field
 /// when logging in.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/login"))]
+#[tracing::instrument]
 pub async fn get_login_types_route() -> ConduitResult<get_login_types::Response> {
-    Ok(get_login_types::Response::new(vec![get_login_types::LoginType::Password]).into())
+    Ok(
+        get_login_types::Response::new(vec![get_login_types::LoginType::Password(
+            Default::default(),
+        )])
+        .into(),
+    )
 }
 
 /// # `POST /_matrix/client/r0/login`
@@ -35,49 +48,74 @@ pub async fn get_login_types_route() -> ConduitResult<get_login_types::Response>
     feature = "conduit_bin",
     post("/_matrix/client/r0/login", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn login_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<login::Request<'_>>,
 ) -> ConduitResult<login::Response> {
     // Validate login method
-    let user_id =
-        // TODO: Other login methods
-        if let (login::IncomingUserInfo::MatrixId(username), login::IncomingLoginInfo::Password { password }) =
-            (&body.user, &body.login_info)
-        {
-            let user_id = UserId::parse_with_server_name(username.to_string(), db.globals.server_name())
-                .map_err(|_| Error::BadRequest(
-                    ErrorKind::InvalidUsername,
-                    "Username is invalid."
-                ))?;
-            let hash = db.users.password_hash(&user_id)?
-                .ok_or(Error::BadRequest(
-                    ErrorKind::Forbidden,
-                    "Wrong username or password."
-                ))?;
+    // TODO: Other login methods
+    let user_id = match &body.login_info {
+        login::IncomingLoginInfo::Password {
+            identifier,
+            password,
+        } => {
+            let username = if let login::IncomingUserIdentifier::MatrixId(matrix_id) = identifier {
+                matrix_id
+            } else {
+                return Err(Error::BadRequest(ErrorKind::Forbidden, "Bad login type."));
+            };
+            let user_id =
+                UserId::parse_with_server_name(username.to_owned(), db.globals.server_name())
+                    .map_err(|_| {
+                        Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid.")
+                    })?;
+            let hash = db.users.password_hash(&user_id)?.ok_or(Error::BadRequest(
+                ErrorKind::Forbidden,
+                "Wrong username or password.",
+            ))?;
 
             if hash.is_empty() {
                 return Err(Error::BadRequest(
                     ErrorKind::UserDeactivated,
-                    "The user has been deactivated"
+                    "The user has been deactivated",
                 ));
             }
 
-            let hash_matches =
-                argon2::verify_encoded(&hash, password.as_bytes()).unwrap_or(false);
+            let hash_matches = argon2::verify_encoded(&hash, password.as_bytes()).unwrap_or(false);
 
             if !hash_matches {
-                return Err(Error::BadRequest(ErrorKind::Forbidden, "Wrong username or password."));
+                return Err(Error::BadRequest(
+                    ErrorKind::Forbidden,
+                    "Wrong username or password.",
+                ));
             }
 
             user_id
-        } else {
-            return Err(Error::BadRequest(ErrorKind::Forbidden, "Bad login type."));
-        };
+        }
+        login::IncomingLoginInfo::Token { token } => {
+            if let Some(jwt_decoding_key) = db.globals.jwt_decoding_key() {
+                let token = jsonwebtoken::decode::<Claims>(
+                    &token,
+                    &jwt_decoding_key,
+                    &jsonwebtoken::Validation::default(),
+                )
+                .map_err(|_| Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid."))?;
+                let username = token.claims.sub;
+                UserId::parse_with_server_name(username, db.globals.server_name()).map_err(
+                    |_| Error::BadRequest(ErrorKind::InvalidUsername, "Username is invalid."),
+                )?
+            } else {
+                return Err(Error::BadRequest(
+                    ErrorKind::Unknown,
+                    "Token login is not supported (server has no jwt decoding key).",
+                ));
+            }
+        }
+    };
 
     // Generate new device id if the user didn't specify one
     let device_id = body
-        .body
         .device_id
         .clone()
         .unwrap_or_else(|| utils::random_string(DEVICE_ID_LENGTH).into());
@@ -85,18 +123,27 @@ pub async fn login_route(
     // Generate a new token for the device
     let token = utils::random_string(TOKEN_LENGTH);
 
-    // TODO: Don't always create a new device
-    // Add device
-    db.users.create_device(
-        &user_id,
-        &device_id,
-        &token,
-        body.initial_device_display_name.clone(),
-    )?;
+    // Determine if device_id was provided and exists in the db for this user
+    let device_exists = body.device_id.as_ref().map_or(false, |device_id| {
+        db.users
+            .all_device_ids(&user_id)
+            .any(|x| x.as_ref().map_or(false, |v| v == device_id))
+    });
+
+    if device_exists {
+        db.users.set_token(&user_id, &device_id, &token)?;
+    } else {
+        db.users.create_device(
+            &user_id,
+            &device_id,
+            &token,
+            body.initial_device_display_name.clone(),
+        )?;
+    }
 
     info!("{} logged in", user_id);
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(login::Response {
         user_id,
@@ -118,8 +165,9 @@ pub async fn login_route(
     feature = "conduit_bin",
     post("/_matrix/client/r0/logout", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn logout_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<logout::Request>,
 ) -> ConduitResult<logout::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -127,7 +175,7 @@ pub async fn logout_route(
 
     db.users.remove_device(&sender_user, sender_device)?;
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(logout::Response::new().into())
 }
@@ -145,19 +193,18 @@ pub async fn logout_route(
     feature = "conduit_bin",
     post("/_matrix/client/r0/logout/all", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn logout_all_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<logout_all::Request>,
 ) -> ConduitResult<logout_all::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    for device_id in db.users.all_device_ids(sender_user) {
-        if let Ok(device_id) = device_id {
-            db.users.remove_device(&sender_user, &device_id)?;
-        }
+    for device_id in db.users.all_device_ids(sender_user).flatten() {
+        db.users.remove_device(&sender_user, &device_id)?;
     }
 
-    db.flush().await?;
+    db.flush()?;
 
     Ok(logout_all::Response::new().into())
 }

src/client_server/state.rs

@@ -1,17 +1,21 @@
-use super::State;
-use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Result, Ruma};
+use std::sync::Arc;
+
+use crate::{
+    database::DatabaseGuard, pdu::PduBuilder, ConduitResult, Database, Error, Result, Ruma,
+};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::state::{
-            get_state_events, get_state_events_for_empty_key, get_state_events_for_key,
-            send_state_event_for_empty_key, send_state_event_for_key,
-        },
+        r0::state::{get_state_events, get_state_events_for_key, send_state_event},
     },
     events::{
-        room::history_visibility::{HistoryVisibility, HistoryVisibilityEventContent},
-        AnyStateEventContent, EventContent, EventType,
+        room::{
+            canonical_alias::CanonicalAliasEventContent,
+            history_visibility::{HistoryVisibility, HistoryVisibilityEventContent},
+        },
+        AnyStateEventContent, EventType,
     },
+    serde::Raw,
     EventId, RoomId, UserId,
 };
 
@@ -22,82 +26,61 @@ use rocket::{get, put};
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn send_state_event_for_key_route(
-    db: State<'_, Database>,
-    body: Ruma<send_state_event_for_key::Request<'_>>,
-) -> ConduitResult<send_state_event_for_key::Response> {
+    db: DatabaseGuard,
+    body: Ruma<send_state_event::Request<'_>>,
+) -> ConduitResult<send_state_event::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    let content = serde_json::from_str::<serde_json::Value>(
-        body.json_body
-            .as_ref()
-            .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?
-            .get(),
-    )
-    .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?;
-
     let event_id = send_state_event_for_key_helper(
         &db,
         sender_user,
-        &body.content,
-        content,
         &body.room_id,
-        Some(body.state_key.to_owned()),
+        EventType::from(&body.event_type),
+        &body.body.body, // Yes, I hate it too
+        body.state_key.to_owned(),
     )
     .await?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(send_state_event_for_key::Response { event_id }.into())
+    Ok(send_state_event::Response { event_id }.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/state/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn send_state_event_for_empty_key_route(
-    db: State<'_, Database>,
-    body: Ruma<send_state_event_for_empty_key::Request<'_>>,
-) -> ConduitResult<send_state_event_for_empty_key::Response> {
-    // This just calls send_state_event_for_key_route
-    let Ruma {
-        body,
-        sender_user,
-        sender_device: _,
-        json_body,
-    } = body;
-
-    let json = serde_json::from_str::<serde_json::Value>(
-        json_body
-            .as_ref()
-            .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?
-            .get(),
-    )
-    .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?;
+    db: DatabaseGuard,
+    body: Ruma<send_state_event::Request<'_>>,
+) -> ConduitResult<send_state_event::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     let event_id = send_state_event_for_key_helper(
         &db,
-        sender_user
-            .as_ref()
-            .expect("no user for send state empty key rout"),
-        &body.content,
-        json,
+        sender_user,
         &body.room_id,
-        Some("".into()),
+        EventType::from(&body.event_type),
+        &body.body.body,
+        body.state_key.to_owned(),
     )
     .await?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(send_state_event_for_empty_key::Response { event_id }.into())
+    Ok(send_state_event::Response { event_id }.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/state", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_state_events_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_state_events::Request<'_>>,
 ) -> ConduitResult<get_state_events::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -109,8 +92,8 @@ pub async fn get_state_events_route(
         && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
-                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
+                .map(|event| {
+                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
                         .map_err(|_| {
                             Error::bad_database(
                                 "Invalid room history visibility event in database.",
@@ -142,8 +125,9 @@ pub async fn get_state_events_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_state_events_for_key_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_state_events_for_key::Request<'_>>,
 ) -> ConduitResult<get_state_events_for_key::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -155,8 +139,8 @@ pub async fn get_state_events_for_key_route(
         && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
-                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
+                .map(|event| {
+                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
                         .map_err(|_| {
                             Error::bad_database(
                                 "Invalid room history visibility event in database.",
@@ -179,11 +163,10 @@ pub async fn get_state_events_for_key_route(
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "State event not found.",
-        ))?
-        .1;
+        ))?;
 
     Ok(get_state_events_for_key::Response {
-        content: serde_json::value::to_raw_value(&event.content)
+        content: serde_json::from_value(event.content.clone())
             .map_err(|_| Error::bad_database("Invalid event content in database"))?,
     }
     .into())
@@ -193,10 +176,11 @@ pub async fn get_state_events_for_key_route(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/state/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_state_events_for_empty_key_route(
-    db: State<'_, Database>,
-    body: Ruma<get_state_events_for_empty_key::Request<'_>>,
-) -> ConduitResult<get_state_events_for_empty_key::Response> {
+    db: DatabaseGuard,
+    body: Ruma<get_state_events_for_key::Request<'_>>,
+) -> ConduitResult<get_state_events_for_key::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     #[allow(clippy::blocks_in_if_conditions)]
@@ -206,8 +190,8 @@ pub async fn get_state_events_for_empty_key_route(
         && !matches!(
             db.rooms
                 .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
-                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
+                .map(|event| {
+                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
                         .map_err(|_| {
                             Error::bad_database(
                                 "Invalid room history visibility event in database.",
@@ -230,11 +214,10 @@ pub async fn get_state_events_for_empty_key_route(
         .ok_or(Error::BadRequest(
             ErrorKind::NotFound,
             "State event not found.",
-        ))?
-        .1;
+        ))?;
 
-    Ok(get_state_events_for_empty_key::Response {
-        content: serde_json::value::to_raw_value(&event)
+    Ok(get_state_events_for_key::Response {
+        content: serde_json::from_value(event.content.clone())
             .map_err(|_| Error::bad_database("Invalid event content in database"))?,
     }
     .into())
@@ -243,17 +226,19 @@ pub async fn get_state_events_for_empty_key_route(
 pub async fn send_state_event_for_key_helper(
     db: &Database,
     sender: &UserId,
-    content: &AnyStateEventContent,
-    json: serde_json::Value,
     room_id: &RoomId,
-    state_key: Option<String>,
+    event_type: EventType,
+    json: &Raw<AnyStateEventContent>,
+    state_key: String,
 ) -> Result<EventId> {
     let sender_user = sender;
 
-    if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content {
+    if let Ok(canonical_alias) =
+        serde_json::from_str::<CanonicalAliasEventContent>(json.json().get())
+    {
         let mut aliases = canonical_alias.alt_aliases.clone();
 
-        if let Some(alias) = canonical_alias.alias.clone() {
+        if let Some(alias) = canonical_alias.alias {
             aliases.push(alias);
         }
 
@@ -274,20 +259,28 @@ pub async fn send_state_event_for_key_helper(
         }
     }
 
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
     let event_id = db.rooms.build_and_append_pdu(
         PduBuilder {
-            event_type: content.event_type().into(),
-            content: json,
+            event_type,
+            content: serde_json::from_str(json.json().get()).expect("content is valid json"),
             unsigned: None,
-            state_key,
+            state_key: Some(state_key),
             redacts: None,
         },
         &sender_user,
         &room_id,
-        &db.globals,
-        &db.sending,
-        &db.admin,
-        &db.account_data,
+        &db,
+        &state_lock,
     )?;
 
     Ok(event_id)

src/client_server/tag.rs

@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Ruma};
 use ruma::{
     api::client::r0::tag::{create_tag, delete_tag, get_tags},
     events::EventType,
@@ -13,8 +12,9 @@ use rocket::{delete, get, put};
     feature = "conduit_bin",
     put("/_matrix/client/r0/user/<_>/rooms/<_>/tags/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn update_tag_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<create_tag::Request<'_>>,
 ) -> ConduitResult<create_tag::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -30,7 +30,7 @@ pub async fn update_tag_route(
     tags_event
         .content
         .tags
-        .insert(body.tag.to_string(), body.tag_info.clone());
+        .insert(body.tag.clone().into(), body.tag_info.clone());
 
     db.account_data.update(
         Some(&body.room_id),
@@ -40,17 +40,18 @@ pub async fn update_tag_route(
         &db.globals,
     )?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(create_tag::Response.into())
+    Ok(create_tag::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/r0/user/<_>/rooms/<_>/tags/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn delete_tag_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<delete_tag::Request<'_>>,
 ) -> ConduitResult<delete_tag::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -63,7 +64,7 @@ pub async fn delete_tag_route(
                 tags: BTreeMap::new(),
             },
         });
-    tags_event.content.tags.remove(&body.tag);
+    tags_event.content.tags.remove(&body.tag.clone().into());
 
     db.account_data.update(
         Some(&body.room_id),
@@ -73,17 +74,18 @@ pub async fn delete_tag_route(
         &db.globals,
     )?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(delete_tag::Response.into())
+    Ok(delete_tag::Response {}.into())
 }
 
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/user/<_>/rooms/<_>/tags", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn get_tags_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<get_tags::Request<'_>>,
 ) -> ConduitResult<get_tags::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");

src/client_server/thirdparty.rs

@@ -1,7 +1,6 @@
 use crate::ConduitResult;
 use ruma::api::client::r0::thirdparty::get_protocols;
 
-use log::warn;
 #[cfg(feature = "conduit_bin")]
 use rocket::get;
 use std::collections::BTreeMap;
@@ -10,8 +9,9 @@ use std::collections::BTreeMap;
     feature = "conduit_bin",
     get("/_matrix/client/r0/thirdparty/protocols")
 )]
+#[tracing::instrument]
 pub async fn get_protocols_route() -> ConduitResult<get_protocols::Response> {
-    warn!("TODO: get_protocols_route");
+    // TODO
     Ok(get_protocols::Response {
         protocols: BTreeMap::new(),
     }

src/client_server/to_device.rs

@@ -1,8 +1,13 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
-use ruma::api::client::{
-    error::ErrorKind,
-    r0::to_device::{self, send_event_to_device},
+use std::collections::BTreeMap;
+
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
+use ruma::{
+    api::{
+        client::{error::ErrorKind, r0::to_device::send_event_to_device},
+        federation::{self, transactions::edu::DirectDeviceContent},
+    },
+    events::EventType,
+    to_device::DeviceIdOrAllDevices,
 };
 
 #[cfg(feature = "conduit_bin")]
@@ -12,14 +17,17 @@ use rocket::put;
     feature = "conduit_bin",
     put("/_matrix/client/r0/sendToDevice/<_>/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn send_event_to_device_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<send_event_to_device::Request<'_>>,
 ) -> ConduitResult<send_event_to_device::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
-    let sender_device = body.sender_device.as_ref().expect("user is authenticated");
+    let sender_device = body.sender_device.as_deref();
 
+    // TODO: uncomment when https://github.com/vector-im/element-android/issues/3589 is solved
     // Check if this is a new transaction id
+    /*
     if db
         .transaction_ids
         .existing_txnid(sender_user, sender_device, &body.txn_id)?
@@ -27,31 +35,52 @@ pub async fn send_event_to_device_route(
     {
         return Ok(send_event_to_device::Response.into());
     }
+    */
 
     for (target_user_id, map) in &body.messages {
         for (target_device_id_maybe, event) in map {
-            match target_device_id_maybe {
-                to_device::DeviceIdOrAllDevices::DeviceId(target_device_id) => {
-                    db.users.add_to_device_event(
-                        sender_user,
-                        &target_user_id,
-                        &target_device_id,
-                        &body.event_type,
-                        serde_json::from_str(event.get()).map_err(|_| {
-                            Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid")
-                        })?,
-                        &db.globals,
-                    )?
-                }
+            if target_user_id.server_name() != db.globals.server_name() {
+                let mut map = BTreeMap::new();
+                map.insert(target_device_id_maybe.clone(), event.clone());
+                let mut messages = BTreeMap::new();
+                messages.insert(target_user_id.clone(), map);
 
-                to_device::DeviceIdOrAllDevices::AllDevices => {
+                db.sending.send_reliable_edu(
+                    target_user_id.server_name(),
+                    serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice(
+                        DirectDeviceContent {
+                            sender: sender_user.clone(),
+                            ev_type: EventType::from(&body.event_type),
+                            message_id: body.txn_id.clone(),
+                            messages,
+                        },
+                    ))
+                    .expect("DirectToDevice EDU can be serialized"),
+                )?;
+
+                continue;
+            }
+
+            match target_device_id_maybe {
+                DeviceIdOrAllDevices::DeviceId(target_device_id) => db.users.add_to_device_event(
+                    sender_user,
+                    &target_user_id,
+                    &target_device_id,
+                    &body.event_type,
+                    event.deserialize_as().map_err(|_| {
+                        Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid")
+                    })?,
+                    &db.globals,
+                )?,
+
+                DeviceIdOrAllDevices::AllDevices => {
                     for target_device_id in db.users.all_device_ids(&target_user_id) {
                         db.users.add_to_device_event(
                             sender_user,
                             &target_user_id,
                             &target_device_id?,
                             &body.event_type,
-                            serde_json::from_str(event.get()).map_err(|_| {
+                            event.deserialize_as().map_err(|_| {
                                 Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid")
                             })?,
                             &db.globals,
@@ -66,7 +95,7 @@ pub async fn send_event_to_device_route(
     db.transaction_ids
         .add_txnid(sender_user, sender_device, &body.txn_id, &[])?;
 
-    db.flush().await?;
+    db.flush()?;
 
-    Ok(send_event_to_device::Response.into())
+    Ok(send_event_to_device::Response {}.into())
 }

src/client_server/typing.rs

@@ -1,5 +1,4 @@
-use super::State;
-use crate::{utils, ConduitResult, Database, Ruma};
+use crate::{database::DatabaseGuard, utils, ConduitResult, Ruma};
 use create_typing_event::Typing;
 use ruma::api::client::r0::typing::create_typing_event;
 
@@ -10,8 +9,9 @@ use rocket::put;
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/typing/<_>", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub fn create_typing_event_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<create_typing_event::Request<'_>>,
 ) -> ConduitResult<create_typing_event::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -29,5 +29,5 @@ pub fn create_typing_event_route(
             .typing_remove(&sender_user, &body.room_id, &db.globals)?;
     }
 
-    Ok(create_typing_event::Response.into())
+    Ok(create_typing_event::Response {}.into())
 }

src/client_server/unversioned.rs

@@ -15,6 +15,7 @@ use rocket::get;
 /// Note: Unstable features are used while developing new features. Clients should avoid using
 /// unstable features in their stable releases
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/versions"))]
+#[tracing::instrument]
 pub async fn get_supported_versions_route() -> ConduitResult<get_supported_versions::Response> {
     let mut resp =
         get_supported_versions::Response::new(vec!["r0.5.0".to_owned(), "r0.6.0".to_owned()]);

src/client_server/user_directory.rs

@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Ruma};
 use ruma::api::client::r0::user_directory::search_users;
 
 #[cfg(feature = "conduit_bin")]
@@ -9,8 +8,9 @@ use rocket::post;
     feature = "conduit_bin",
     post("/_matrix/client/r0/user_directory/search", data = "<body>")
 )]
+#[tracing::instrument(skip(db, body))]
 pub async fn search_users_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
     body: Ruma<search_users::Request<'_>>,
 ) -> ConduitResult<search_users::Response> {
     let limit = u64::from(body.limit) as usize;
@@ -18,9 +18,6 @@ pub async fn search_users_route(
     let mut users = db.users.iter().filter_map(|user_id| {
         // Filter out buggy users (they should not exist, but you never know...)
         let user_id = user_id.ok()?;
-        if db.users.is_deactivated(&user_id).ok()? {
-            return None;
-        }
 
         let user = search_users::User {
             user_id: user_id.clone(),
@@ -28,13 +25,22 @@ pub async fn search_users_route(
             avatar_url: db.users.avatar_url(&user_id).ok()?,
         };
 
-        if !user.user_id.to_string().contains(&body.search_term)
-            && user
-                .display_name
-                .as_ref()
-                .filter(|name| name.contains(&body.search_term))
-                .is_none()
-        {
+        let user_id_matches = user
+            .user_id
+            .to_string()
+            .to_lowercase()
+            .contains(&body.search_term.to_lowercase());
+
+        let user_displayname_matches = user
+            .display_name
+            .as_ref()
+            .filter(|name| {
+                name.to_lowercase()
+                    .contains(&body.search_term.to_lowercase())
+            })
+            .is_some();
+
+        if !user_id_matches && !user_displayname_matches {
             return None;
         }
 

src/client_server/voip.rs

@@ -1,13 +1,18 @@
-use crate::{ConduitResult, Error};
-use ruma::api::client::{error::ErrorKind, r0::message::send_message_event};
+use crate::ConduitResult;
+use ruma::api::client::r0::voip::get_turn_server_info;
+use std::time::Duration;
 
 #[cfg(feature = "conduit_bin")]
 use rocket::get;
 
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/voip/turnServer"))]
-pub async fn turn_server_route() -> ConduitResult<send_message_event::Response> {
-    Err(Error::BadRequest(
-        ErrorKind::NotFound,
-        "There is no turn server yet.",
-    ))
+#[tracing::instrument]
+pub async fn turn_server_route() -> ConduitResult<get_turn_server_info::Response> {
+    Ok(get_turn_server_info::Response {
+        username: "".to_owned(),
+        password: "".to_owned(),
+        uris: Vec::new(),
+        ttl: Duration::from_secs(60 * 60 * 24),
+    }
+    .into())
 }

src/database.rs

@@ -1,54 +1,138 @@
+pub mod abstraction;
+
 pub mod account_data;
 pub mod admin;
+pub mod appservice;
 pub mod globals;
 pub mod key_backups;
 pub mod media;
+pub mod proxy;
+pub mod pusher;
 pub mod rooms;
 pub mod sending;
 pub mod transaction_ids;
 pub mod uiaa;
 pub mod users;
 
-use crate::{Error, Result};
+use crate::{utils, Error, Result};
+use abstraction::DatabaseEngine;
 use directories::ProjectDirs;
-use futures::StreamExt;
-use log::info;
-use rocket::futures::{self, channel::mpsc};
-use ruma::{DeviceId, ServerName, UserId};
-use serde::Deserialize;
-use std::{convert::TryInto, fs::remove_dir_all};
+use lru_cache::LruCache;
+use rocket::{
+    futures::{channel::mpsc, stream::FuturesUnordered, StreamExt},
+    outcome::{try_outcome, IntoOutcome},
+    request::{FromRequest, Request},
+    Shutdown, State,
+};
+use ruma::{DeviceId, RoomId, ServerName, UserId};
+use serde::{de::IgnoredAny, Deserialize};
+use std::{
+    collections::{BTreeMap, HashMap},
+    convert::TryFrom,
+    fs::{self, remove_dir_all},
+    io::Write,
+    ops::Deref,
+    path::Path,
+    sync::{Arc, Mutex, RwLock},
+};
+use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore};
+use tracing::{debug, error, warn};
 
-#[derive(Clone, Deserialize)]
+use self::proxy::ProxyConfig;
+
+#[derive(Clone, Debug, Deserialize)]
 pub struct Config {
-    #[serde(default = "default_server_name")]
     server_name: Box<ServerName>,
-    database_path: Option<String>,
-    #[serde(default = "default_cache_capacity")]
-    cache_capacity: u64,
+    database_path: String,
+    #[serde(default = "default_db_cache_capacity_mb")]
+    db_cache_capacity_mb: f64,
+    #[serde(default = "default_sqlite_wal_clean_second_interval")]
+    sqlite_wal_clean_second_interval: u32,
     #[serde(default = "default_max_request_size")]
     max_request_size: u32,
+    #[serde(default = "default_max_concurrent_requests")]
+    max_concurrent_requests: u16,
+    #[serde(default = "true_fn")]
+    allow_registration: bool,
+    #[serde(default = "true_fn")]
+    allow_encryption: bool,
+    #[serde(default = "false_fn")]
+    allow_federation: bool,
+    #[serde(default = "false_fn")]
+    pub allow_jaeger: bool,
+    #[serde(default = "false_fn")]
+    pub tracing_flame: bool,
     #[serde(default)]
-    registration_disabled: bool,
-    #[serde(default)]
-    encryption_disabled: bool,
-    #[serde(default)]
-    federation_enabled: bool,
+    proxy: ProxyConfig,
+    jwt_secret: Option<String>,
+    #[serde(default = "Vec::new")]
+    trusted_servers: Vec<Box<ServerName>>,
+    #[serde(default = "default_log")]
+    pub log: String,
+
+    #[serde(flatten)]
+    catchall: BTreeMap<String, IgnoredAny>,
 }
 
-fn default_server_name() -> Box<ServerName> {
-    "localhost".try_into().expect("")
+const DEPRECATED_KEYS: &[&str] = &["cache_capacity"];
+
+impl Config {
+    pub fn warn_deprecated(&self) {
+        let mut was_deprecated = false;
+        for key in self
+            .catchall
+            .keys()
+            .filter(|key| DEPRECATED_KEYS.iter().any(|s| s == key))
+        {
+            warn!("Config parameter {} is deprecated", key);
+            was_deprecated = true;
+        }
+
+        if was_deprecated {
+            warn!("Read conduit documentation and check your configuration if any new configuration parameters should be adjusted");
+        }
+    }
 }
 
-fn default_cache_capacity() -> u64 {
-    1024 * 1024 * 1024
+fn false_fn() -> bool {
+    false
+}
+
+fn true_fn() -> bool {
+    true
+}
+
+fn default_db_cache_capacity_mb() -> f64 {
+    200.0
+}
+
+fn default_sqlite_wal_clean_second_interval() -> u32 {
+    15 * 60 // every 15 minutes
 }
 
 fn default_max_request_size() -> u32 {
     20 * 1024 * 1024 // Default to 20 MB
 }
 
-#[derive(Clone)]
+fn default_max_concurrent_requests() -> u16 {
+    100
+}
+
+fn default_log() -> String {
+    "info,state_res=warn,rocket=off,_=off,sled=off".to_owned()
+}
+
+#[cfg(feature = "sled")]
+pub type Engine = abstraction::sled::Engine;
+
+#[cfg(feature = "sqlite")]
+pub type Engine = abstraction::sqlite::Engine;
+
+#[cfg(feature = "heed")]
+pub type Engine = abstraction::heed::Engine;
+
 pub struct Database {
+    _db: Arc<Engine>,
     pub globals: globals::Globals,
     pub users: users::Users,
     pub uiaa: uiaa::Uiaa,
@@ -59,7 +143,8 @@ pub struct Database {
     pub transaction_ids: transaction_ids::TransactionIds,
     pub sending: sending::Sending,
     pub admin: admin::Admin,
-    pub _db: sled::Db,
+    pub appservice: appservice::Appservice,
+    pub pusher: pusher::PushData,
 }
 
 impl Database {
@@ -75,121 +160,321 @@ impl Database {
         Ok(())
     }
 
+    fn check_sled_or_sqlite_db(config: &Config) -> Result<()> {
+        #[cfg(feature = "backend_sqlite")]
+        {
+            let path = Path::new(&config.database_path);
+
+            let sled_exists = path.join("db").exists();
+            let sqlite_exists = path.join("conduit.db").exists();
+            if sled_exists {
+                if sqlite_exists {
+                    // most likely an in-place directory, only warn
+                    warn!("Both sled and sqlite databases are detected in database directory");
+                    warn!("Currently running from the sqlite database, but consider removing sled database files to free up space")
+                } else {
+                    error!(
+                        "Sled database detected, conduit now uses sqlite for database operations"
+                    );
+                    error!("This database must be converted to sqlite, go to https://github.com/ShadowJonathan/conduit_toolbox#conduit_sled_to_sqlite");
+                    return Err(Error::bad_config(
+                        "sled database detected, migrate to sqlite",
+                    ));
+                }
+            }
+        }
+
+        Ok(())
+    }
+
     /// Load an existing database or create a new one.
-    pub async fn load_or_create(config: Config) -> Result<Self> {
-        let path = config
-            .database_path
-            .clone()
-            .map(Ok::<_, Error>)
-            .unwrap_or_else(|| {
-                let path = ProjectDirs::from("xyz", "koesters", "conduit")
-                    .ok_or_else(|| {
-                        Error::bad_config("The OS didn't return a valid home directory path.")
-                    })?
-                    .data_dir()
-                    .join(config.server_name.as_str());
+    pub async fn load_or_create(config: &Config) -> Result<Arc<TokioRwLock<Self>>> {
+        Self::check_sled_or_sqlite_db(&config)?;
 
-                Ok(path
-                    .to_str()
-                    .ok_or_else(|| Error::bad_config("Database path contains invalid unicode."))?
-                    .to_owned())
-            })?;
+        let builder = Engine::open(&config)?;
 
-        let db = sled::Config::default()
-            .path(&path)
-            .cache_capacity(config.cache_capacity)
-            .print_profile_on_drop(true)
-            .open()?;
-
-        info!("Opened sled database at {}", path);
+        if config.max_request_size < 1024 {
+            eprintln!("ERROR: Max request size is less than 1KB. Please increase it.");
+        }
 
         let (admin_sender, admin_receiver) = mpsc::unbounded();
+        let (sending_sender, sending_receiver) = mpsc::unbounded();
 
-        let db = Self {
-            globals: globals::Globals::load(db.open_tree("global")?, config).await?,
+        let db = Arc::new(TokioRwLock::from(Self {
+            _db: builder.clone(),
             users: users::Users {
-                userid_password: db.open_tree("userid_password")?,
-                userid_displayname: db.open_tree("userid_displayname")?,
-                userid_avatarurl: db.open_tree("userid_avatarurl")?,
-                userdeviceid_token: db.open_tree("userdeviceid_token")?,
-                userdeviceid_metadata: db.open_tree("userdeviceid_metadata")?,
-                token_userdeviceid: db.open_tree("token_userdeviceid")?,
-                onetimekeyid_onetimekeys: db.open_tree("onetimekeyid_onetimekeys")?,
-                userid_lastonetimekeyupdate: db.open_tree("userid_lastonetimekeyupdate")?,
-                keychangeid_userid: db.open_tree("devicekeychangeid_userid")?,
-                keyid_key: db.open_tree("keyid_key")?,
-                userid_masterkeyid: db.open_tree("userid_masterkeyid")?,
-                userid_selfsigningkeyid: db.open_tree("userid_selfsigningkeyid")?,
-                userid_usersigningkeyid: db.open_tree("userid_usersigningkeyid")?,
-                todeviceid_events: db.open_tree("todeviceid_events")?,
+                userid_password: builder.open_tree("userid_password")?,
+                userid_displayname: builder.open_tree("userid_displayname")?,
+                userid_avatarurl: builder.open_tree("userid_avatarurl")?,
+                userid_blurhash: builder.open_tree("userid_blurhash")?,
+                userdeviceid_token: builder.open_tree("userdeviceid_token")?,
+                userdeviceid_metadata: builder.open_tree("userdeviceid_metadata")?,
+                userid_devicelistversion: builder.open_tree("userid_devicelistversion")?,
+                token_userdeviceid: builder.open_tree("token_userdeviceid")?,
+                onetimekeyid_onetimekeys: builder.open_tree("onetimekeyid_onetimekeys")?,
+                userid_lastonetimekeyupdate: builder.open_tree("userid_lastonetimekeyupdate")?,
+                keychangeid_userid: builder.open_tree("keychangeid_userid")?,
+                keyid_key: builder.open_tree("keyid_key")?,
+                userid_masterkeyid: builder.open_tree("userid_masterkeyid")?,
+                userid_selfsigningkeyid: builder.open_tree("userid_selfsigningkeyid")?,
+                userid_usersigningkeyid: builder.open_tree("userid_usersigningkeyid")?,
+                todeviceid_events: builder.open_tree("todeviceid_events")?,
             },
             uiaa: uiaa::Uiaa {
-                userdeviceid_uiaainfo: db.open_tree("userdeviceid_uiaainfo")?,
+                userdevicesessionid_uiaainfo: builder.open_tree("userdevicesessionid_uiaainfo")?,
+                userdevicesessionid_uiaarequest: builder
+                    .open_tree("userdevicesessionid_uiaarequest")?,
             },
             rooms: rooms::Rooms {
                 edus: rooms::RoomEdus {
-                    readreceiptid_readreceipt: db.open_tree("readreceiptid_readreceipt")?,
-                    roomuserid_privateread: db.open_tree("roomuserid_privateread")?, // "Private" read receipt
-                    roomuserid_lastprivatereadupdate: db
-                        .open_tree("roomid_lastprivatereadupdate")?,
-                    typingid_userid: db.open_tree("typingid_userid")?,
-                    roomid_lasttypingupdate: db.open_tree("roomid_lasttypingupdate")?,
-                    presenceid_presence: db.open_tree("presenceid_presence")?,
-                    userid_lastpresenceupdate: db.open_tree("userid_lastpresenceupdate")?,
+                    readreceiptid_readreceipt: builder.open_tree("readreceiptid_readreceipt")?,
+                    roomuserid_privateread: builder.open_tree("roomuserid_privateread")?, // "Private" read receipt
+                    roomuserid_lastprivatereadupdate: builder
+                        .open_tree("roomuserid_lastprivatereadupdate")?,
+                    typingid_userid: builder.open_tree("typingid_userid")?,
+                    roomid_lasttypingupdate: builder.open_tree("roomid_lasttypingupdate")?,
+                    presenceid_presence: builder.open_tree("presenceid_presence")?,
+                    userid_lastpresenceupdate: builder.open_tree("userid_lastpresenceupdate")?,
                 },
-                pduid_pdu: db.open_tree("pduid_pdu")?,
-                eventid_pduid: db.open_tree("eventid_pduid")?,
-                roomid_pduleaves: db.open_tree("roomid_pduleaves")?,
+                pduid_pdu: builder.open_tree("pduid_pdu")?,
+                eventid_pduid: builder.open_tree("eventid_pduid")?,
+                roomid_pduleaves: builder.open_tree("roomid_pduleaves")?,
 
-                alias_roomid: db.open_tree("alias_roomid")?,
-                aliasid_alias: db.open_tree("alias_roomid")?,
-                publicroomids: db.open_tree("publicroomids")?,
+                alias_roomid: builder.open_tree("alias_roomid")?,
+                aliasid_alias: builder.open_tree("aliasid_alias")?,
+                publicroomids: builder.open_tree("publicroomids")?,
 
-                tokenids: db.open_tree("tokenids")?,
+                tokenids: builder.open_tree("tokenids")?,
 
-                roomserverids: db.open_tree("roomserverids")?,
-                userroomid_joined: db.open_tree("userroomid_joined")?,
-                roomuserid_joined: db.open_tree("roomuserid_joined")?,
-                roomuseroncejoinedids: db.open_tree("roomuseroncejoinedids")?,
-                userroomid_invited: db.open_tree("userroomid_invited")?,
-                roomuserid_invited: db.open_tree("roomuserid_invited")?,
-                userroomid_left: db.open_tree("userroomid_left")?,
+                roomserverids: builder.open_tree("roomserverids")?,
+                serverroomids: builder.open_tree("serverroomids")?,
+                userroomid_joined: builder.open_tree("userroomid_joined")?,
+                roomuserid_joined: builder.open_tree("roomuserid_joined")?,
+                roomid_joinedcount: builder.open_tree("roomid_joinedcount")?,
+                roomuseroncejoinedids: builder.open_tree("roomuseroncejoinedids")?,
+                userroomid_invitestate: builder.open_tree("userroomid_invitestate")?,
+                roomuserid_invitecount: builder.open_tree("roomuserid_invitecount")?,
+                userroomid_leftstate: builder.open_tree("userroomid_leftstate")?,
+                roomuserid_leftcount: builder.open_tree("roomuserid_leftcount")?,
 
-                stateid_pduid: db.open_tree("stateid_pduid")?,
-                pduid_statehash: db.open_tree("pduid_statehash")?,
-                roomid_statehash: db.open_tree("roomid_statehash")?,
+                userroomid_notificationcount: builder.open_tree("userroomid_notificationcount")?,
+                userroomid_highlightcount: builder.open_tree("userroomid_highlightcount")?,
+
+                statekey_shortstatekey: builder.open_tree("statekey_shortstatekey")?,
+                stateid_shorteventid: builder.open_tree("stateid_shorteventid")?,
+                eventid_shorteventid: builder.open_tree("eventid_shorteventid")?,
+                shorteventid_eventid: builder.open_tree("shorteventid_eventid")?,
+                shorteventid_shortstatehash: builder.open_tree("shorteventid_shortstatehash")?,
+                roomid_shortstatehash: builder.open_tree("roomid_shortstatehash")?,
+                statehash_shortstatehash: builder.open_tree("statehash_shortstatehash")?,
+
+                eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?,
+                referencedevents: builder.open_tree("referencedevents")?,
+                pdu_cache: Mutex::new(LruCache::new(100_000)),
+                auth_chain_cache: Mutex::new(LruCache::new(100_000)),
+                shorteventid_cache: Mutex::new(LruCache::new(1_000_000)),
             },
             account_data: account_data::AccountData {
-                roomuserdataid_accountdata: db.open_tree("roomuserdataid_accountdata")?,
+                roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,
+                roomusertype_roomuserdataid: builder.open_tree("roomusertype_roomuserdataid")?,
             },
             media: media::Media {
-                mediaid_file: db.open_tree("mediaid_file")?,
+                mediaid_file: builder.open_tree("mediaid_file")?,
             },
             key_backups: key_backups::KeyBackups {
-                backupid_algorithm: db.open_tree("backupid_algorithm")?,
-                backupid_etag: db.open_tree("backupid_etag")?,
-                backupkeyid_backup: db.open_tree("backupkeyid_backupmetadata")?,
+                backupid_algorithm: builder.open_tree("backupid_algorithm")?,
+                backupid_etag: builder.open_tree("backupid_etag")?,
+                backupkeyid_backup: builder.open_tree("backupkeyid_backup")?,
             },
             transaction_ids: transaction_ids::TransactionIds {
-                userdevicetxnid_response: db.open_tree("userdevicetxnid_response")?,
+                userdevicetxnid_response: builder.open_tree("userdevicetxnid_response")?,
             },
             sending: sending::Sending {
-                servernamepduids: db.open_tree("servernamepduids")?,
-                servercurrentpdus: db.open_tree("servercurrentpdus")?,
+                servername_educount: builder.open_tree("servername_educount")?,
+                servernameevent_data: builder.open_tree("servernameevent_data")?,
+                servercurrentevent_data: builder.open_tree("servercurrentevent_data")?,
+                maximum_requests: Arc::new(Semaphore::new(config.max_concurrent_requests as usize)),
+                sender: sending_sender,
             },
             admin: admin::Admin {
                 sender: admin_sender,
             },
-            _db: db,
-        };
+            appservice: appservice::Appservice {
+                cached_registrations: Arc::new(RwLock::new(HashMap::new())),
+                id_appserviceregistrations: builder.open_tree("id_appserviceregistrations")?,
+            },
+            pusher: pusher::PushData {
+                senderkey_pusher: builder.open_tree("senderkey_pusher")?,
+            },
+            globals: globals::Globals::load(
+                builder.open_tree("global")?,
+                builder.open_tree("server_signingkeys")?,
+                config.clone(),
+            )?,
+        }));
 
-        db.admin.start_handler(db.clone(), admin_receiver);
+        {
+            let db = db.read().await;
+            // MIGRATIONS
+            // TODO: database versions of new dbs should probably not be 0
+            if db.globals.database_version()? < 1 {
+                for (roomserverid, _) in db.rooms.roomserverids.iter() {
+                    let mut parts = roomserverid.split(|&b| b == 0xff);
+                    let room_id = parts.next().expect("split always returns one element");
+                    let servername = match parts.next() {
+                        Some(s) => s,
+                        None => {
+                            error!("Migration: Invalid roomserverid in db.");
+                            continue;
+                        }
+                    };
+                    let mut serverroomid = servername.to_vec();
+                    serverroomid.push(0xff);
+                    serverroomid.extend_from_slice(room_id);
+
+                    db.rooms.serverroomids.insert(&serverroomid, &[])?;
+                }
+
+                db.globals.bump_database_version(1)?;
+
+                println!("Migration: 0 -> 1 finished");
+            }
+
+            if db.globals.database_version()? < 2 {
+                // We accidentally inserted hashed versions of "" into the db instead of just ""
+                for (userid, password) in db.users.userid_password.iter() {
+                    let password = utils::string_from_bytes(&password);
+
+                    let empty_hashed_password = password.map_or(false, |password| {
+                        argon2::verify_encoded(&password, b"").unwrap_or(false)
+                    });
+
+                    if empty_hashed_password {
+                        db.users.userid_password.insert(&userid, b"")?;
+                    }
+                }
+
+                db.globals.bump_database_version(2)?;
+
+                println!("Migration: 1 -> 2 finished");
+            }
+
+            if db.globals.database_version()? < 3 {
+                // Move media to filesystem
+                for (key, content) in db.media.mediaid_file.iter() {
+                    if content.is_empty() {
+                        continue;
+                    }
+
+                    let path = db.globals.get_media_file(&key);
+                    let mut file = fs::File::create(path)?;
+                    file.write_all(&content)?;
+                    db.media.mediaid_file.insert(&key, &[])?;
+                }
+
+                db.globals.bump_database_version(3)?;
+
+                println!("Migration: 2 -> 3 finished");
+            }
+
+            if db.globals.database_version()? < 4 {
+                // Add federated users to db as deactivated
+                for our_user in db.users.iter() {
+                    let our_user = our_user?;
+                    if db.users.is_deactivated(&our_user)? {
+                        continue;
+                    }
+                    for room in db.rooms.rooms_joined(&our_user) {
+                        for user in db.rooms.room_members(&room?) {
+                            let user = user?;
+                            if user.server_name() != db.globals.server_name() {
+                                println!("Migration: Creating user {}", user);
+                                db.users.create(&user, None)?;
+                            }
+                        }
+                    }
+                }
+
+                db.globals.bump_database_version(4)?;
+
+                println!("Migration: 3 -> 4 finished");
+            }
+
+            if db.globals.database_version()? < 5 {
+                // Upgrade user data store
+                for (roomuserdataid, _) in db.account_data.roomuserdataid_accountdata.iter() {
+                    let mut parts = roomuserdataid.split(|&b| b == 0xff);
+                    let room_id = parts.next().unwrap();
+                    let user_id = parts.next().unwrap();
+                    let event_type = roomuserdataid.rsplit(|&b| b == 0xff).next().unwrap();
+
+                    let mut key = room_id.to_vec();
+                    key.push(0xff);
+                    key.extend_from_slice(user_id);
+                    key.push(0xff);
+                    key.extend_from_slice(event_type);
+
+                    db.account_data
+                        .roomusertype_roomuserdataid
+                        .insert(&key, &roomuserdataid)?;
+                }
+
+                db.globals.bump_database_version(5)?;
+
+                println!("Migration: 4 -> 5 finished");
+            }
+
+            if db.globals.database_version()? < 6 {
+                // TODO update to 6
+                // Set room member count
+                for (roomid, _) in db.rooms.roomid_shortstatehash.iter() {
+                    let room_id =
+                        RoomId::try_from(utils::string_from_bytes(&roomid).unwrap()).unwrap();
+
+                    db.rooms.update_joined_count(&room_id)?;
+                }
+
+                db.globals.bump_database_version(6)?;
+
+                println!("Migration: 5 -> 6 finished");
+            }
+        }
+
+        let guard = db.read().await;
+
+        // This data is probably outdated
+        guard.rooms.edus.presenceid_presence.clear()?;
+
+        guard.admin.start_handler(Arc::clone(&db), admin_receiver);
+        guard
+            .sending
+            .start_handler(Arc::clone(&db), sending_receiver);
+
+        drop(guard);
+
+        #[cfg(feature = "sqlite")]
+        {
+            Self::start_wal_clean_task(Arc::clone(&db), &config).await;
+        }
 
         Ok(db)
     }
 
+    #[cfg(feature = "conduit_bin")]
+    pub async fn start_on_shutdown_tasks(db: Arc<TokioRwLock<Self>>, shutdown: Shutdown) {
+        use tracing::info;
+
+        tokio::spawn(async move {
+            shutdown.await;
+
+            info!(target: "shutdown-sync", "Received shutdown notification, notifying sync helpers...");
+
+            db.read().await.globals.rotate.fire();
+        });
+    }
+
     pub async fn watch(&self, user_id: &UserId, device_id: &DeviceId) {
-        let userid_bytes = user_id.to_string().as_bytes().to_vec();
+        let userid_bytes = user_id.as_bytes().to_vec();
         let mut userid_prefix = userid_bytes.clone();
         userid_prefix.push(0xff);
 
@@ -197,7 +482,7 @@ impl Database {
         userdeviceid_prefix.extend_from_slice(device_id.as_bytes());
         userdeviceid_prefix.push(0xff);
 
-        let mut futures = futures::stream::FuturesUnordered::new();
+        let mut futures = FuturesUnordered::new();
 
         // Return when *any* user changed his key
         // TODO: only send for user they share a room with
@@ -208,12 +493,26 @@ impl Database {
         );
 
         futures.push(self.rooms.userroomid_joined.watch_prefix(&userid_prefix));
-        futures.push(self.rooms.userroomid_invited.watch_prefix(&userid_prefix));
-        futures.push(self.rooms.userroomid_left.watch_prefix(&userid_prefix));
+        futures.push(
+            self.rooms
+                .userroomid_invitestate
+                .watch_prefix(&userid_prefix),
+        );
+        futures.push(self.rooms.userroomid_leftstate.watch_prefix(&userid_prefix));
+        futures.push(
+            self.rooms
+                .userroomid_notificationcount
+                .watch_prefix(&userid_prefix),
+        );
+        futures.push(
+            self.rooms
+                .userroomid_highlightcount
+                .watch_prefix(&userid_prefix),
+        );
 
         // Events for rooms we are in
         for room_id in self.rooms.rooms_joined(user_id).filter_map(|r| r.ok()) {
-            let roomid_bytes = room_id.to_string().as_bytes().to_vec();
+            let roomid_bytes = room_id.as_bytes().to_vec();
             let mut roomid_prefix = roomid_bytes.clone();
             roomid_prefix.push(0xff);
 
@@ -244,7 +543,7 @@ impl Database {
 
             futures.push(
                 self.account_data
-                    .roomuserdataid_accountdata
+                    .roomusertype_roomuserdataid
                     .watch_prefix(&roomuser_prefix),
             );
         }
@@ -254,7 +553,7 @@ impl Database {
 
         futures.push(
             self.account_data
-                .roomuserdataid_accountdata
+                .roomusertype_roomuserdataid
                 .watch_prefix(&globaluserdata_prefix),
         );
 
@@ -268,12 +567,97 @@ impl Database {
                 .watch_prefix(&userid_bytes),
         );
 
+        futures.push(Box::pin(self.globals.rotate.watch()));
+
         // Wait until one of them finds something
         futures.next().await;
     }
 
-    pub async fn flush(&self) -> Result<()> {
-        self._db.flush_async().await?;
-        Ok(())
+    #[tracing::instrument(skip(self))]
+    pub fn flush(&self) -> Result<()> {
+        let start = std::time::Instant::now();
+
+        let res = self._db.flush();
+
+        debug!("flush: took {:?}", start.elapsed());
+
+        res
+    }
+
+    #[cfg(feature = "sqlite")]
+    #[tracing::instrument(skip(self))]
+    pub fn flush_wal(&self) -> Result<()> {
+        self._db.flush_wal()
+    }
+
+    #[cfg(feature = "sqlite")]
+    #[tracing::instrument(skip(db, config))]
+    pub async fn start_wal_clean_task(db: Arc<TokioRwLock<Self>>, config: &Config) {
+        use tokio::time::interval;
+
+        #[cfg(unix)]
+        use tokio::signal::unix::{signal, SignalKind};
+        use tracing::info;
+
+        use std::time::{Duration, Instant};
+
+        let timer_interval = Duration::from_secs(config.sqlite_wal_clean_second_interval as u64);
+
+        tokio::spawn(async move {
+            let mut i = interval(timer_interval);
+            #[cfg(unix)]
+            let mut s = signal(SignalKind::hangup()).unwrap();
+
+            loop {
+                #[cfg(unix)]
+                tokio::select! {
+                    _ = i.tick() => {
+                        info!("wal-trunc: Timer ticked");
+                    }
+                    _ = s.recv() => {
+                        info!("wal-trunc: Received SIGHUP");
+                    }
+                };
+                #[cfg(not(unix))]
+                {
+                    i.tick().await;
+                    info!("wal-trunc: Timer ticked")
+                }
+
+                let start = Instant::now();
+                if let Err(e) = db.read().await.flush_wal() {
+                    error!("wal-trunc: Errored: {}", e);
+                } else {
+                    info!("wal-trunc: Flushed in {:?}", start.elapsed());
+                }
+            }
+        });
+    }
+}
+
+pub struct DatabaseGuard(OwnedRwLockReadGuard<Database>);
+
+impl Deref for DatabaseGuard {
+    type Target = OwnedRwLockReadGuard<Database>;
+
+    fn deref(&self) -> &Self::Target {
+        &self.0
+    }
+}
+
+#[rocket::async_trait]
+impl<'r> FromRequest<'r> for DatabaseGuard {
+    type Error = ();
+
+    async fn from_request(req: &'r Request<'_>) -> rocket::request::Outcome<Self, ()> {
+        let db = try_outcome!(req.guard::<&State<Arc<TokioRwLock<Database>>>>().await);
+
+        Ok(DatabaseGuard(Arc::clone(&db).read_owned().await)).or_forward(())
+    }
+}
+
+impl From<OwnedRwLockReadGuard<Database>> for DatabaseGuard {
+    fn from(val: OwnedRwLockReadGuard<Database>) -> Self {
+        Self(val)
     }
 }

src/database/abstraction.rs

@@ -0,0 +1,53 @@
+use super::Config;
+use crate::Result;
+
+use std::{future::Future, pin::Pin, sync::Arc};
+
+#[cfg(feature = "sled")]
+pub mod sled;
+
+#[cfg(feature = "sqlite")]
+pub mod sqlite;
+
+#[cfg(feature = "heed")]
+pub mod heed;
+
+pub trait DatabaseEngine: Sized {
+    fn open(config: &Config) -> Result<Arc<Self>>;
+    fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>>;
+    fn flush(self: &Arc<Self>) -> Result<()>;
+}
+
+pub trait Tree: Send + Sync {
+    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>>;
+
+    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()>;
+    fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()>;
+
+    fn remove(&self, key: &[u8]) -> Result<()>;
+
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
+
+    fn iter_from<'a>(
+        &'a self,
+        from: &[u8],
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
+
+    fn increment(&self, key: &[u8]) -> Result<Vec<u8>>;
+
+    fn scan_prefix<'a>(
+        &'a self,
+        prefix: Vec<u8>,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
+
+    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
+
+    fn clear(&self) -> Result<()> {
+        for (key, _) in self.iter() {
+            self.remove(&key)?;
+        }
+
+        Ok(())
+    }
+}

src/database/abstraction/heed.rs

@@ -0,0 +1,240 @@
+use super::super::Config;
+use crossbeam::channel::{bounded, Sender as ChannelSender};
+use threadpool::ThreadPool;
+
+use crate::{Error, Result};
+use std::{
+    collections::HashMap,
+    future::Future,
+    pin::Pin,
+    sync::{Arc, Mutex, RwLock},
+};
+use tokio::sync::oneshot::Sender;
+
+use super::{DatabaseEngine, Tree};
+
+type TupleOfBytes = (Vec<u8>, Vec<u8>);
+
+pub struct Engine {
+    env: heed::Env,
+    iter_pool: Mutex<ThreadPool>,
+}
+
+pub struct EngineTree {
+    engine: Arc<Engine>,
+    tree: Arc<heed::UntypedDatabase>,
+    watchers: RwLock<HashMap<Vec<u8>, Vec<Sender<()>>>>,
+}
+
+fn convert_error(error: heed::Error) -> Error {
+    Error::HeedError {
+        error: error.to_string(),
+    }
+}
+
+impl DatabaseEngine for Engine {
+    fn open(config: &Config) -> Result<Arc<Self>> {
+        let mut env_builder = heed::EnvOpenOptions::new();
+        env_builder.map_size(1024 * 1024 * 1024 * 1024); // 1 Terabyte
+        env_builder.max_readers(126);
+        env_builder.max_dbs(128);
+        unsafe {
+            env_builder.flag(heed::flags::Flags::MdbWriteMap);
+            env_builder.flag(heed::flags::Flags::MdbMapAsync);
+        }
+
+        Ok(Arc::new(Engine {
+            env: env_builder
+                .open(&config.database_path)
+                .map_err(convert_error)?,
+            iter_pool: Mutex::new(ThreadPool::new(10)),
+        }))
+    }
+
+    fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> {
+        // Creates the db if it doesn't exist already
+        Ok(Arc::new(EngineTree {
+            engine: Arc::clone(self),
+            tree: Arc::new(
+                self.env
+                    .create_database(Some(name))
+                    .map_err(convert_error)?,
+            ),
+            watchers: RwLock::new(HashMap::new()),
+        }))
+    }
+
+    fn flush(self: &Arc<Self>) -> Result<()> {
+        self.env.force_sync().map_err(convert_error)?;
+        Ok(())
+    }
+}
+
+impl EngineTree {
+    #[tracing::instrument(skip(self, tree, from, backwards))]
+    fn iter_from_thread(
+        &self,
+        tree: Arc<heed::UntypedDatabase>,
+        from: Vec<u8>,
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = TupleOfBytes> + Send + Sync> {
+        let (s, r) = bounded::<TupleOfBytes>(100);
+        let engine = Arc::clone(&self.engine);
+
+        let lock = self.engine.iter_pool.lock().await;
+        if lock.active_count() < lock.max_count() {
+            lock.execute(move || {
+                iter_from_thread_work(tree, &engine.env.read_txn().unwrap(), from, backwards, &s);
+            });
+        } else {
+            std::thread::spawn(move || {
+                iter_from_thread_work(tree, &engine.env.read_txn().unwrap(), from, backwards, &s);
+            });
+        }
+
+        Box::new(r.into_iter())
+    }
+}
+
+#[tracing::instrument(skip(tree, txn, from, backwards))]
+fn iter_from_thread_work(
+    tree: Arc<heed::UntypedDatabase>,
+    txn: &heed::RoTxn<'_>,
+    from: Vec<u8>,
+    backwards: bool,
+    s: &ChannelSender<(Vec<u8>, Vec<u8>)>,
+) {
+    if backwards {
+        for (k, v) in tree.rev_range(txn, ..=&*from).unwrap().map(|r| r.unwrap()) {
+            if s.send((k.to_vec(), v.to_vec())).is_err() {
+                return;
+            }
+        }
+    } else {
+        if from.is_empty() {
+            for (k, v) in tree.iter(txn).unwrap().map(|r| r.unwrap()) {
+                if s.send((k.to_vec(), v.to_vec())).is_err() {
+                    return;
+                }
+            }
+        } else {
+            for (k, v) in tree.range(txn, &*from..).unwrap().map(|r| r.unwrap()) {
+                if s.send((k.to_vec(), v.to_vec())).is_err() {
+                    return;
+                }
+            }
+        }
+    }
+}
+
+impl Tree for EngineTree {
+    #[tracing::instrument(skip(self, key))]
+    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        let txn = self.engine.env.read_txn().map_err(convert_error)?;
+        Ok(self
+            .tree
+            .get(&txn, &key)
+            .map_err(convert_error)?
+            .map(|s| s.to_vec()))
+    }
+
+    #[tracing::instrument(skip(self, key, value))]
+    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
+        let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
+        self.tree
+            .put(&mut txn, &key, &value)
+            .map_err(convert_error)?;
+        txn.commit().map_err(convert_error)?;
+
+        let watchers = self.watchers.read().unwrap();
+        let mut triggered = Vec::new();
+
+        for length in 0..=key.len() {
+            if watchers.contains_key(&key[..length]) {
+                triggered.push(&key[..length]);
+            }
+        }
+
+        drop(watchers);
+
+        if !triggered.is_empty() {
+            let mut watchers = self.watchers.write().unwrap();
+            for prefix in triggered {
+                if let Some(txs) = watchers.remove(prefix) {
+                    for tx in txs {
+                        let _ = tx.send(());
+                    }
+                }
+            }
+        };
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, key))]
+    fn remove(&self, key: &[u8]) -> Result<()> {
+        let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
+        self.tree.delete(&mut txn, &key).map_err(convert_error)?;
+        txn.commit().map_err(convert_error)?;
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> {
+        self.iter_from(&[], false)
+    }
+
+    #[tracing::instrument(skip(self, from, backwards))]
+    fn iter_from(
+        &self,
+        from: &[u8],
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send> {
+        self.iter_from_thread(Arc::clone(&self.tree), from.to_vec(), backwards)
+    }
+
+    #[tracing::instrument(skip(self, key))]
+    fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
+        let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
+
+        let old = self.tree.get(&txn, &key).map_err(convert_error)?;
+        let new =
+            crate::utils::increment(old.as_deref()).expect("utils::increment always returns Some");
+
+        self.tree
+            .put(&mut txn, &key, &&*new)
+            .map_err(convert_error)?;
+
+        txn.commit().map_err(convert_error)?;
+
+        Ok(new)
+    }
+
+    #[tracing::instrument(skip(self, prefix))]
+    fn scan_prefix<'a>(
+        &'a self,
+        prefix: Vec<u8>,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> {
+        Box::new(
+            self.iter_from(&prefix, false)
+                .take_while(move |(key, _)| key.starts_with(&prefix)),
+        )
+    }
+
+    #[tracing::instrument(skip(self, prefix))]
+    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
+        let (tx, rx) = tokio::sync::oneshot::channel();
+
+        self.watchers
+            .write()
+            .unwrap()
+            .entry(prefix.to_vec())
+            .or_default()
+            .push(tx);
+
+        Box::pin(async move {
+            // Tx is never destroyed
+            rx.await.unwrap();
+        })
+    }
+}

src/database/abstraction/sled.rs

@@ -0,0 +1,128 @@
+use super::super::Config;
+use crate::{utils, Result};
+use std::{future::Future, pin::Pin, sync::Arc};
+use tracing::warn;
+
+use super::{DatabaseEngine, Tree};
+
+pub struct Engine(sled::Db);
+
+pub struct SledEngineTree(sled::Tree);
+
+impl DatabaseEngine for Engine {
+    fn open(config: &Config) -> Result<Arc<Self>> {
+        Ok(Arc::new(Engine(
+            sled::Config::default()
+                .path(&config.database_path)
+                .cache_capacity((config.db_cache_capacity_mb * 1024.0 * 1024.0) as u64)
+                .use_compression(true)
+                .open()?,
+        )))
+    }
+
+    fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> {
+        Ok(Arc::new(SledEngineTree(self.0.open_tree(name)?)))
+    }
+
+    fn flush(self: &Arc<Self>) -> Result<()> {
+        Ok(()) // noop
+    }
+}
+
+impl Tree for SledEngineTree {
+    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        Ok(self.0.get(key)?.map(|v| v.to_vec()))
+    }
+
+    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
+        self.0.insert(key, value)?;
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, iter))]
+    fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
+        for (key, value) in iter {
+            self.0.insert(key, value)?;
+        }
+
+        Ok(())
+    }
+
+    fn remove(&self, key: &[u8]) -> Result<()> {
+        self.0.remove(key)?;
+        Ok(())
+    }
+
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
+        Box::new(
+            self.0
+                .iter()
+                .filter_map(|r| {
+                    if let Err(e) = &r {
+                        warn!("Error: {}", e);
+                    }
+                    r.ok()
+                })
+                .map(|(k, v)| (k.to_vec().into(), v.to_vec().into())),
+        )
+    }
+
+    fn iter_from(
+        &self,
+        from: &[u8],
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)>> {
+        let iter = if backwards {
+            self.0.range(..=from)
+        } else {
+            self.0.range(from..)
+        };
+
+        let iter = iter
+            .filter_map(|r| {
+                if let Err(e) = &r {
+                    warn!("Error: {}", e);
+                }
+                r.ok()
+            })
+            .map(|(k, v)| (k.to_vec().into(), v.to_vec().into()));
+
+        if backwards {
+            Box::new(iter.rev())
+        } else {
+            Box::new(iter)
+        }
+    }
+
+    fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
+        Ok(self
+            .0
+            .update_and_fetch(key, utils::increment)
+            .map(|o| o.expect("increment always sets a value").to_vec())?)
+    }
+
+    fn scan_prefix<'a>(
+        &'a self,
+        prefix: Vec<u8>,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
+        let iter = self
+            .0
+            .scan_prefix(prefix)
+            .filter_map(|r| {
+                if let Err(e) = &r {
+                    warn!("Error: {}", e);
+                }
+                r.ok()
+            })
+            .map(|(k, v)| (k.to_vec().into(), v.to_vec().into()));
+
+        Box::new(iter)
+    }
+
+    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
+        let prefix = prefix.to_vec();
+        Box::pin(async move {
+            self.0.watch_prefix(prefix).await;
+        })
+    }
+}

src/database/abstraction/sqlite.rs

@@ -0,0 +1,352 @@
+use super::{DatabaseEngine, Tree};
+use crate::{database::Config, Result};
+use parking_lot::{Mutex, MutexGuard, RwLock};
+use rusqlite::{Connection, DatabaseName::Main, OptionalExtension};
+use std::{
+    cell::RefCell,
+    collections::HashMap,
+    future::Future,
+    path::{Path, PathBuf},
+    pin::Pin,
+    sync::Arc,
+    time::{Duration, Instant},
+};
+use tokio::sync::oneshot::Sender;
+use tracing::debug;
+
+thread_local! {
+    static READ_CONNECTION: RefCell<Option<&'static Connection>> = RefCell::new(None);
+}
+
+struct PreparedStatementIterator<'a> {
+    pub iterator: Box<dyn Iterator<Item = TupleOfBytes> + 'a>,
+    pub statement_ref: NonAliasingBox<rusqlite::Statement<'a>>,
+}
+
+impl Iterator for PreparedStatementIterator<'_> {
+    type Item = TupleOfBytes;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.iterator.next()
+    }
+}
+
+struct NonAliasingBox<T>(*mut T);
+impl<T> Drop for NonAliasingBox<T> {
+    fn drop(&mut self) {
+        unsafe { Box::from_raw(self.0) };
+    }
+}
+
+pub struct Engine {
+    writer: Mutex<Connection>,
+
+    path: PathBuf,
+    cache_size_per_thread: u32,
+}
+
+impl Engine {
+    fn prepare_conn(path: &Path, cache_size_kb: u32) -> Result<Connection> {
+        let conn = Connection::open(&path)?;
+
+        conn.pragma_update(Some(Main), "page_size", &32768)?;
+        conn.pragma_update(Some(Main), "journal_mode", &"WAL")?;
+        conn.pragma_update(Some(Main), "synchronous", &"NORMAL")?;
+        conn.pragma_update(Some(Main), "cache_size", &(-i64::from(cache_size_kb)))?;
+        conn.pragma_update(Some(Main), "wal_autocheckpoint", &0)?;
+
+        Ok(conn)
+    }
+
+    fn write_lock(&self) -> MutexGuard<'_, Connection> {
+        self.writer.lock()
+    }
+
+    fn read_lock(&self) -> &'static Connection {
+        READ_CONNECTION.with(|cell| {
+            let connection = &mut cell.borrow_mut();
+
+            if (*connection).is_none() {
+                let c = Box::leak(Box::new(
+                    Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap(),
+                ));
+                **connection = Some(c);
+            }
+
+            connection.unwrap()
+        })
+    }
+
+    pub fn flush_wal(self: &Arc<Self>) -> Result<()> {
+        self.write_lock()
+            .pragma_update(Some(Main), "wal_checkpoint", &"TRUNCATE")?;
+        Ok(())
+    }
+}
+
+impl DatabaseEngine for Engine {
+    fn open(config: &Config) -> Result<Arc<Self>> {
+        let path = Path::new(&config.database_path).join("conduit.db");
+
+        // calculates cache-size per permanent connection
+        // 1. convert MB to KiB
+        // 2. divide by permanent connections
+        // 3. round down to nearest integer
+        let cache_size_per_thread: u32 =
+            ((config.db_cache_capacity_mb * 1024.0) / (num_cpus::get().max(1) + 1) as f64) as u32;
+
+        let writer = Mutex::new(Self::prepare_conn(&path, cache_size_per_thread)?);
+
+        let arc = Arc::new(Engine {
+            writer,
+            path,
+            cache_size_per_thread,
+        });
+
+        Ok(arc)
+    }
+
+    fn open_tree(self: &Arc<Self>, name: &str) -> Result<Arc<dyn Tree>> {
+        self.write_lock().execute(&format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", name), [])?;
+
+        Ok(Arc::new(SqliteTable {
+            engine: Arc::clone(self),
+            name: name.to_owned(),
+            watchers: RwLock::new(HashMap::new()),
+        }))
+    }
+
+    fn flush(self: &Arc<Self>) -> Result<()> {
+        // we enabled PRAGMA synchronous=normal, so this should not be necessary
+        Ok(())
+    }
+}
+
+pub struct SqliteTable {
+    engine: Arc<Engine>,
+    name: String,
+    watchers: RwLock<HashMap<Vec<u8>, Vec<Sender<()>>>>,
+}
+
+type TupleOfBytes = (Vec<u8>, Vec<u8>);
+
+impl SqliteTable {
+    #[tracing::instrument(skip(self, guard, key))]
+    fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        Ok(guard
+            .prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())?
+            .query_row([key], |row| row.get(0))
+            .optional()?)
+    }
+
+    #[tracing::instrument(skip(self, guard, key, value))]
+    fn insert_with_guard(&self, guard: &Connection, key: &[u8], value: &[u8]) -> Result<()> {
+        guard.execute(
+            format!(
+                "INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)",
+                self.name
+            )
+            .as_str(),
+            [key, value],
+        )?;
+        Ok(())
+    }
+}
+
+impl Tree for SqliteTable {
+    #[tracing::instrument(skip(self, key))]
+    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        self.get_with_guard(&self.engine.read_lock(), key)
+    }
+
+    #[tracing::instrument(skip(self, key, value))]
+    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
+        let guard = self.engine.write_lock();
+        self.insert_with_guard(&guard, key, value)?;
+        drop(guard);
+
+        let watchers = self.watchers.read();
+        let mut triggered = Vec::new();
+
+        for length in 0..=key.len() {
+            if watchers.contains_key(&key[..length]) {
+                triggered.push(&key[..length]);
+            }
+        }
+
+        drop(watchers);
+
+        if !triggered.is_empty() {
+            let mut watchers = self.watchers.write();
+            for prefix in triggered {
+                if let Some(txs) = watchers.remove(prefix) {
+                    for tx in txs {
+                        let _ = tx.send(());
+                    }
+                }
+            }
+        };
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, iter))]
+    fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
+        let guard = self.engine.write_lock();
+
+        guard.execute("BEGIN", [])?;
+        for (key, value) in iter {
+            self.insert_with_guard(&guard, &key, &value)?;
+        }
+        guard.execute("COMMIT", [])?;
+
+        drop(guard);
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, key))]
+    fn remove(&self, key: &[u8]) -> Result<()> {
+        let guard = self.engine.write_lock();
+
+        guard.execute(
+            format!("DELETE FROM {} WHERE key = ?", self.name).as_str(),
+            [key],
+        )?;
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
+        let guard = self.engine.read_lock();
+
+        let statement = Box::leak(Box::new(
+            guard
+                .prepare(&format!(
+                    "SELECT key, value FROM {} ORDER BY key ASC",
+                    &self.name
+                ))
+                .unwrap(),
+        ));
+
+        let statement_ref = NonAliasingBox(statement);
+
+        let iterator = Box::new(
+            statement
+                .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
+                .unwrap()
+                .map(|r| r.unwrap()),
+        );
+
+        Box::new(PreparedStatementIterator {
+            iterator,
+            statement_ref,
+        })
+    }
+
+    #[tracing::instrument(skip(self, from, backwards))]
+    fn iter_from<'a>(
+        &'a self,
+        from: &[u8],
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
+        let guard = self.engine.read_lock();
+        let from = from.to_vec(); // TODO change interface?
+
+        if backwards {
+            let statement = Box::leak(Box::new(
+                guard
+                    .prepare(&format!(
+                        "SELECT key, value FROM {} WHERE key <= ? ORDER BY key DESC",
+                        &self.name
+                    ))
+                    .unwrap(),
+            ));
+
+            let statement_ref = NonAliasingBox(statement);
+
+            let iterator = Box::new(
+                statement
+                    .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
+                    .unwrap()
+                    .map(|r| r.unwrap()),
+            );
+            Box::new(PreparedStatementIterator {
+                iterator,
+                statement_ref,
+            })
+        } else {
+            let statement = Box::leak(Box::new(
+                guard
+                    .prepare(&format!(
+                        "SELECT key, value FROM {} WHERE key >= ? ORDER BY key ASC",
+                        &self.name
+                    ))
+                    .unwrap(),
+            ));
+
+            let statement_ref = NonAliasingBox(statement);
+
+            let iterator = Box::new(
+                statement
+                    .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
+                    .unwrap()
+                    .map(|r| r.unwrap()),
+            );
+
+            Box::new(PreparedStatementIterator {
+                iterator,
+                statement_ref,
+            })
+        }
+    }
+
+    #[tracing::instrument(skip(self, key))]
+    fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
+        let guard = self.engine.write_lock();
+
+        let old = self.get_with_guard(&guard, key)?;
+
+        let new =
+            crate::utils::increment(old.as_deref()).expect("utils::increment always returns Some");
+
+        self.insert_with_guard(&guard, key, &new)?;
+
+        Ok(new)
+    }
+
+    #[tracing::instrument(skip(self, prefix))]
+    fn scan_prefix<'a>(&'a self, prefix: Vec<u8>) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
+        Box::new(
+            self.iter_from(&prefix, false)
+                .take_while(move |(key, _)| key.starts_with(&prefix)),
+        )
+    }
+
+    #[tracing::instrument(skip(self, prefix))]
+    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
+        let (tx, rx) = tokio::sync::oneshot::channel();
+
+        self.watchers
+            .write()
+            .entry(prefix.to_vec())
+            .or_default()
+            .push(tx);
+
+        Box::pin(async move {
+            // Tx is never destroyed
+            rx.await.unwrap();
+        })
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn clear(&self) -> Result<()> {
+        debug!("clear: running");
+        self.engine
+            .write_lock()
+            .execute(format!("DELETE FROM {}", self.name).as_str(), [])?;
+        debug!("clear: ran");
+        Ok(())
+    }
+}

src/database/account_data.rs

@@ -1,21 +1,23 @@
 use crate::{utils, Error, Result};
 use ruma::{
     api::client::error::ErrorKind,
-    events::{AnyEvent as EduEvent, EventType},
+    events::{AnyEphemeralRoomEvent, EventType},
     serde::Raw,
     RoomId, UserId,
 };
 use serde::{de::DeserializeOwned, Serialize};
-use sled::IVec;
-use std::{collections::HashMap, convert::TryFrom};
+use std::{collections::HashMap, convert::TryFrom, sync::Arc};
+
+use super::abstraction::Tree;
 
-#[derive(Clone)]
 pub struct AccountData {
-    pub(super) roomuserdataid_accountdata: sled::Tree, // RoomUserDataId = Room + User + Count + Type
+    pub(super) roomuserdataid_accountdata: Arc<dyn Tree>, // RoomUserDataId = Room + User + Count + Type
+    pub(super) roomusertype_roomuserdataid: Arc<dyn Tree>, // RoomUserType = Room + User + Type
 }
 
 impl AccountData {
     /// Places one event in the account data of the user and removes the previous entry.
+    #[tracing::instrument(skip(self, room_id, user_id, event_type, data, globals))]
     pub fn update<T: Serialize>(
         &self,
         room_id: Option<&RoomId>,
@@ -30,19 +32,16 @@ impl AccountData {
             .as_bytes()
             .to_vec();
         prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
+        prefix.extend_from_slice(&user_id.as_bytes());
         prefix.push(0xff);
 
-        // Remove old entry
-        if let Some(previous) = self.find_event(room_id, user_id, &event_type) {
-            let (old_key, _) = previous?;
-            self.roomuserdataid_accountdata.remove(old_key)?;
-        }
+        let mut roomuserdataid = prefix.clone();
+        roomuserdataid.extend_from_slice(&globals.next_count()?.to_be_bytes());
+        roomuserdataid.push(0xff);
+        roomuserdataid.extend_from_slice(&event_type.as_bytes());
 
-        let mut key = prefix;
-        key.extend_from_slice(&globals.next_count()?.to_be_bytes());
-        key.push(0xff);
-        key.extend_from_slice(event_type.to_string().as_bytes());
+        let mut key = prefix.clone();
+        key.extend_from_slice(event_type.as_bytes());
 
         let json = serde_json::to_value(data).expect("all types here can be serialized"); // TODO: maybe add error handling
         if json.get("type").is_none() || json.get("content").is_none() {
@@ -52,34 +51,65 @@ impl AccountData {
             ));
         }
 
-        self.roomuserdataid_accountdata
-            .insert(key, &*json.to_string())?;
+        self.roomuserdataid_accountdata.insert(
+            &roomuserdataid,
+            &serde_json::to_vec(&json).expect("to_vec always works on json values"),
+        )?;
+
+        let prev = self.roomusertype_roomuserdataid.get(&key)?;
+
+        self.roomusertype_roomuserdataid
+            .insert(&key, &roomuserdataid)?;
+
+        // Remove old entry
+        if let Some(prev) = prev {
+            self.roomuserdataid_accountdata.remove(&prev)?;
+        }
 
         Ok(())
     }
 
     /// Searches the account data for a specific kind.
+    #[tracing::instrument(skip(self, room_id, user_id, kind))]
     pub fn get<T: DeserializeOwned>(
         &self,
         room_id: Option<&RoomId>,
         user_id: &UserId,
         kind: EventType,
     ) -> Result<Option<T>> {
-        self.find_event(room_id, user_id, &kind)
-            .map(|r| {
-                let (_, v) = r?;
-                serde_json::from_slice(&v).map_err(|_| Error::bad_database("could not deserialize"))
+        let mut key = room_id
+            .map(|r| r.to_string())
+            .unwrap_or_default()
+            .as_bytes()
+            .to_vec();
+        key.push(0xff);
+        key.extend_from_slice(&user_id.as_bytes());
+        key.push(0xff);
+        key.extend_from_slice(kind.as_ref().as_bytes());
+
+        self.roomusertype_roomuserdataid
+            .get(&key)?
+            .and_then(|roomuserdataid| {
+                self.roomuserdataid_accountdata
+                    .get(&roomuserdataid)
+                    .transpose()
+            })
+            .transpose()?
+            .map(|data| {
+                serde_json::from_slice(&data)
+                    .map_err(|_| Error::bad_database("could not deserialize"))
             })
             .transpose()
     }
 
     /// Returns all changes to the account data that happened after `since`.
+    #[tracing::instrument(skip(self, room_id, user_id, since))]
     pub fn changes_since(
         &self,
         room_id: Option<&RoomId>,
         user_id: &UserId,
         since: u64,
-    ) -> Result<HashMap<EventType, Raw<EduEvent>>> {
+    ) -> Result<HashMap<EventType, Raw<AnyEphemeralRoomEvent>>> {
         let mut userdata = HashMap::new();
 
         let mut prefix = room_id
@@ -88,7 +118,7 @@ impl AccountData {
             .as_bytes()
             .to_vec();
         prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
+        prefix.extend_from_slice(&user_id.as_bytes());
         prefix.push(0xff);
 
         // Skip the data that's exactly at since, because we sent that last time
@@ -97,8 +127,7 @@ impl AccountData {
 
         for r in self
             .roomuserdataid_accountdata
-            .range(&*first_possible..)
-            .filter_map(|r| r.ok())
+            .iter_from(&first_possible, false)
             .take_while(move |(k, _)| k.starts_with(&prefix))
             .map(|(k, v)| {
                 Ok::<_, Error>((
@@ -109,7 +138,7 @@ impl AccountData {
                         .map_err(|_| Error::bad_database("RoomUserData ID in db is invalid."))?,
                     )
                     .map_err(|_| Error::bad_database("RoomUserData ID in db is invalid."))?,
-                    serde_json::from_slice::<Raw<EduEvent>>(&v).map_err(|_| {
+                    serde_json::from_slice::<Raw<AnyEphemeralRoomEvent>>(&v).map_err(|_| {
                         Error::bad_database("Database contains invalid account data.")
                     })?,
                 ))
@@ -121,38 +150,4 @@ impl AccountData {
 
         Ok(userdata)
     }
-
-    fn find_event(
-        &self,
-        room_id: Option<&RoomId>,
-        user_id: &UserId,
-        kind: &EventType,
-    ) -> Option<Result<(IVec, IVec)>> {
-        let mut prefix = room_id
-            .map(|r| r.to_string())
-            .unwrap_or_default()
-            .as_bytes()
-            .to_vec();
-        prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
-        prefix.push(0xff);
-        let kind = kind.clone();
-
-        self.roomuserdataid_accountdata
-            .scan_prefix(prefix)
-            .rev()
-            .find(move |r| {
-                r.as_ref()
-                    .map(|(k, _)| {
-                        k.rsplit(|&b| b == 0xff)
-                            .next()
-                            .map(|current_event_type| {
-                                current_event_type == kind.to_string().as_bytes()
-                            })
-                            .unwrap_or(false)
-                    })
-                    .unwrap_or(false)
-            })
-            .map(|r| Ok(r?))
-    }
 }

src/database/admin.rs

@@ -1,16 +1,21 @@
-use std::convert::{TryFrom, TryInto};
+use std::{
+    convert::{TryFrom, TryInto},
+    sync::Arc,
+};
 
-use crate::pdu::PduBuilder;
-use log::warn;
+use crate::{pdu::PduBuilder, Database};
 use rocket::futures::{channel::mpsc, stream::StreamExt};
 use ruma::{
     events::{room::message, EventType},
     UserId,
 };
-use tokio::select;
+use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard};
+use tracing::warn;
 
 pub enum AdminCommand {
-    SendTextMessage(message::TextMessageEventContent),
+    RegisterAppservice(serde_yaml::Value),
+    ListAppservices,
+    SendMessage(message::MessageEventContent),
 }
 
 #[derive(Clone)]
@@ -21,53 +26,97 @@ pub struct Admin {
 impl Admin {
     pub fn start_handler(
         &self,
-        db: super::Database,
+        db: Arc<RwLock<Database>>,
         mut receiver: mpsc::UnboundedReceiver<AdminCommand>,
     ) {
         tokio::spawn(async move {
             // TODO: Use futures when we have long admin commands
             //let mut futures = FuturesUnordered::new();
 
-            let conduit_user = UserId::try_from(format!("@conduit:{}", db.globals.server_name()))
-                .expect("@conduit:server_name is valid");
+            let guard = db.read().await;
 
-            let conduit_room = db
+            let conduit_user =
+                UserId::try_from(format!("@conduit:{}", guard.globals.server_name()))
+                    .expect("@conduit:server_name is valid");
+
+            let conduit_room = guard
                 .rooms
                 .id_from_alias(
-                    &format!("#admins:{}", db.globals.server_name())
+                    &format!("#admins:{}", guard.globals.server_name())
                         .try_into()
                         .expect("#admins:server_name is a valid room alias"),
                 )
                 .unwrap();
 
-            if conduit_room.is_none() {
-                warn!("Conduit instance does not have an #admins room. Logging to that room will not work.");
-            }
+            let conduit_room = match conduit_room {
+                None => {
+                    warn!("Conduit instance does not have an #admins room. Logging to that room will not work. Restart Conduit after creating a user to fix this.");
+                    return;
+                }
+                Some(r) => r,
+            };
+
+            drop(guard);
+
+            let send_message = |message: message::MessageEventContent,
+                                guard: RwLockReadGuard<'_, Database>,
+                                mutex_lock: &MutexGuard<'_, ()>| {
+                guard
+                    .rooms
+                    .build_and_append_pdu(
+                        PduBuilder {
+                            event_type: EventType::RoomMessage,
+                            content: serde_json::to_value(message)
+                                .expect("event is valid, we just created it"),
+                            unsigned: None,
+                            state_key: None,
+                            redacts: None,
+                        },
+                        &conduit_user,
+                        &conduit_room,
+                        &guard,
+                        mutex_lock,
+                    )
+                    .unwrap();
+            };
 
             loop {
-                select! {
+                tokio::select! {
                     Some(event) = receiver.next() => {
+                        let guard = db.read().await;
+                        let mutex_state = Arc::clone(
+                            guard.globals
+                                .roomid_mutex_state
+                                .write()
+                                .unwrap()
+                                .entry(conduit_room.clone())
+                                .or_default(),
+                        );
+                        let state_lock = mutex_state.lock().await;
+
                         match event {
-                            AdminCommand::SendTextMessage(message) => {
-                                if let Some(conduit_room) = &conduit_room {
-                                    db.rooms.build_and_append_pdu(
-                                        PduBuilder {
-                                            event_type: EventType::RoomMessage,
-                                            content: serde_json::to_value(message).expect("event is valid, we just created it"),
-                                            unsigned: None,
-                                            state_key: None,
-                                            redacts: None,
-                                        },
-                                        &conduit_user,
-                                        &conduit_room,
-                                        &db.globals,
-                                        &db.sending,
-                                        &db.admin,
-                                        &db.account_data,
-                                    ).unwrap();
+                            AdminCommand::RegisterAppservice(yaml) => {
+                                guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error
+                            }
+                            AdminCommand::ListAppservices => {
+                                if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::<Vec<_>>()) {
+                                    let count = appservices.len();
+                                    let output = format!(
+                                        "Appservices ({}): {}",
+                                        count,
+                                        appservices.into_iter().filter_map(|r| r.ok()).collect::<Vec<_>>().join(", ")
+                                    );
+                                    send_message(message::MessageEventContent::text_plain(output), guard, &state_lock);
+                                } else {
+                                    send_message(message::MessageEventContent::text_plain("Failed to get appservices."), guard, &state_lock);
                                 }
                             }
+                            AdminCommand::SendMessage(message) => {
+                                send_message(message, guard, &state_lock);
+                            }
                         }
+
+                        drop(state_lock);
                     }
                 }
             }
@@ -75,6 +124,6 @@ impl Admin {
     }
 
     pub fn send(&self, command: AdminCommand) {
-        self.sender.unbounded_send(command).unwrap()
+        self.sender.unbounded_send(command).unwrap();
     }
 }

src/database/appservice.rs

@@ -0,0 +1,71 @@
+use crate::{utils, Error, Result};
+use std::{
+    collections::HashMap,
+    sync::{Arc, RwLock},
+};
+
+use super::abstraction::Tree;
+
+pub struct Appservice {
+    pub(super) cached_registrations: Arc<RwLock<HashMap<String, serde_yaml::Value>>>,
+    pub(super) id_appserviceregistrations: Arc<dyn Tree>,
+}
+
+impl Appservice {
+    pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> {
+        // TODO: Rumaify
+        let id = yaml.get("id").unwrap().as_str().unwrap();
+        self.id_appserviceregistrations.insert(
+            id.as_bytes(),
+            serde_yaml::to_string(&yaml).unwrap().as_bytes(),
+        )?;
+        self.cached_registrations
+            .write()
+            .unwrap()
+            .insert(id.to_owned(), yaml);
+
+        Ok(())
+    }
+
+    pub fn get_registration(&self, id: &str) -> Result<Option<serde_yaml::Value>> {
+        self.cached_registrations
+            .read()
+            .unwrap()
+            .get(id)
+            .map_or_else(
+                || {
+                    self.id_appserviceregistrations
+                        .get(id.as_bytes())?
+                        .map(|bytes| {
+                            serde_yaml::from_slice(&bytes).map_err(|_| {
+                                Error::bad_database(
+                                    "Invalid registration bytes in id_appserviceregistrations.",
+                                )
+                            })
+                        })
+                        .transpose()
+                },
+                |r| Ok(Some(r.clone())),
+            )
+    }
+
+    pub fn iter_ids(&self) -> Result<impl Iterator<Item = Result<String>> + '_> {
+        Ok(self.id_appserviceregistrations.iter().map(|(id, _)| {
+            utils::string_from_bytes(&id)
+                .map_err(|_| Error::bad_database("Invalid id bytes in id_appserviceregistrations."))
+        }))
+    }
+
+    pub fn all(&self) -> Result<Vec<(String, serde_yaml::Value)>> {
+        self.iter_ids()?
+            .filter_map(|id| id.ok())
+            .map(move |id| {
+                Ok((
+                    id.clone(),
+                    self.get_registration(&id)?
+                        .expect("iter_ids only returns appservices that exist"),
+                ))
+            })
+            .collect()
+    }
+}

src/database/globals.rs

@@ -1,30 +1,137 @@
-use crate::{database::Config, utils, Error, Result};
+use crate::{database::Config, utils, ConduitResult, Error, Result};
+use ruma::{
+    api::{
+        client::r0::sync::sync_events,
+        federation::discovery::{ServerSigningKeys, VerifyKey},
+    },
+    DeviceId, EventId, MilliSecondsSinceUnixEpoch, RoomId, ServerName, ServerSigningKeyId, UserId,
+};
+use rustls::{ServerCertVerifier, WebPKIVerifier};
+use std::{
+    collections::{BTreeMap, HashMap},
+    fs,
+    future::Future,
+    path::PathBuf,
+    sync::{Arc, Mutex, RwLock},
+    time::{Duration, Instant},
+};
+use tokio::sync::{broadcast, watch::Receiver, Mutex as TokioMutex, Semaphore};
+use tracing::{error, info};
 use trust_dns_resolver::TokioAsyncResolver;
-use std::collections::HashMap;
-use log::error;
-use ruma::ServerName;
-use std::sync::Arc;
-use std::sync::RwLock;
 
-pub const COUNTER: &str = "c";
+use super::abstraction::Tree;
+
+pub const COUNTER: &[u8] = b"c";
+
+type WellKnownMap = HashMap<Box<ServerName>, (String, String)>;
+type TlsNameMap = HashMap<String, webpki::DNSName>;
+type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries
+type SyncHandle = (
+    Option<String>,                                         // since
+    Receiver<Option<ConduitResult<sync_events::Response>>>, // rx
+);
 
-#[derive(Clone)]
 pub struct Globals {
-    pub(super) globals: sled::Tree,
+    pub actual_destination_cache: Arc<RwLock<WellKnownMap>>, // actual_destination, host
+    pub tls_name_override: Arc<RwLock<TlsNameMap>>,
+    pub(super) globals: Arc<dyn Tree>,
     config: Config,
     keypair: Arc<ruma::signatures::Ed25519KeyPair>,
     reqwest_client: reqwest::Client,
-    pub actual_destination_cache: Arc<RwLock<HashMap<Box<ServerName>, (String, Option<String>)>>>, // actual_destination, host
     dns_resolver: TokioAsyncResolver,
+    jwt_decoding_key: Option<jsonwebtoken::DecodingKey<'static>>,
+    pub(super) server_signingkeys: Arc<dyn Tree>,
+    pub bad_event_ratelimiter: Arc<RwLock<HashMap<EventId, RateLimitState>>>,
+    pub bad_signature_ratelimiter: Arc<RwLock<HashMap<Vec<String>, RateLimitState>>>,
+    pub servername_ratelimiter: Arc<RwLock<HashMap<Box<ServerName>, Arc<Semaphore>>>>,
+    pub sync_receivers: RwLock<HashMap<(UserId, Box<DeviceId>), SyncHandle>>,
+    pub roomid_mutex_insert: RwLock<HashMap<RoomId, Arc<Mutex<()>>>>,
+    pub roomid_mutex_state: RwLock<HashMap<RoomId, Arc<TokioMutex<()>>>>,
+    pub roomid_mutex_federation: RwLock<HashMap<RoomId, Arc<TokioMutex<()>>>>, // this lock will be held longer
+    pub rotate: RotationHandler,
+}
+
+struct MatrixServerVerifier {
+    inner: WebPKIVerifier,
+    tls_name_override: Arc<RwLock<TlsNameMap>>,
+}
+
+impl ServerCertVerifier for MatrixServerVerifier {
+    #[tracing::instrument(skip(self, roots, presented_certs, dns_name, ocsp_response))]
+    fn verify_server_cert(
+        &self,
+        roots: &rustls::RootCertStore,
+        presented_certs: &[rustls::Certificate],
+        dns_name: webpki::DNSNameRef<'_>,
+        ocsp_response: &[u8],
+    ) -> std::result::Result<rustls::ServerCertVerified, rustls::TLSError> {
+        if let Some(override_name) = self.tls_name_override.read().unwrap().get(dns_name.into()) {
+            let result = self.inner.verify_server_cert(
+                roots,
+                presented_certs,
+                override_name.as_ref(),
+                ocsp_response,
+            );
+            if result.is_ok() {
+                return result;
+            }
+            info!(
+                "Server {:?} is non-compliant, retrying TLS verification with original name",
+                dns_name
+            );
+        }
+        self.inner
+            .verify_server_cert(roots, presented_certs, dns_name, ocsp_response)
+    }
+}
+
+/// Handles "rotation" of long-polling requests. "Rotation" in this context is similar to "rotation" of log files and the like.
+///
+/// This is utilized to have sync workers return early and release read locks on the database.
+pub struct RotationHandler(broadcast::Sender<()>, broadcast::Receiver<()>);
+
+impl RotationHandler {
+    pub fn new() -> Self {
+        let (s, r) = broadcast::channel::<()>(1);
+
+        Self(s, r)
+    }
+
+    pub fn watch(&self) -> impl Future<Output = ()> {
+        let mut r = self.0.subscribe();
+
+        async move {
+            let _ = r.recv().await;
+        }
+    }
+
+    pub fn fire(&self) {
+        let _ = self.0.send(());
+    }
+}
+
+impl Default for RotationHandler {
+    fn default() -> Self {
+        Self::new()
+    }
 }
 
 impl Globals {
-    pub async fn load(globals: sled::Tree, config: Config) -> Result<Self> {
-        let bytes = &*globals
-            .update_and_fetch("keypair", utils::generate_keypair)?
-            .expect("utils::generate_keypair always returns Some");
+    pub fn load(
+        globals: Arc<dyn Tree>,
+        server_signingkeys: Arc<dyn Tree>,
+        config: Config,
+    ) -> Result<Self> {
+        let keypair_bytes = globals.get(b"keypair")?.map_or_else(
+            || {
+                let keypair = utils::generate_keypair();
+                globals.insert(b"keypair", &keypair)?;
+                Ok::<_, Error>(keypair)
+            },
+            |s| Ok(s.to_vec()),
+        )?;
 
-        let mut parts = bytes.splitn(2, |&b| b == 0xff);
+        let mut parts = keypair_bytes.splitn(2, |&b| b == 0xff);
 
         let keypair = utils::string_from_bytes(
             // 1. version
@@ -41,7 +148,7 @@ impl Globals {
                 .map(|key| (version, key))
         })
         .and_then(|(version, key)| {
-            ruma::signatures::Ed25519KeyPair::new(&key, version)
+            ruma::signatures::Ed25519KeyPair::from_der(&key, version)
                 .map_err(|_| Error::bad_database("Private or public keys are invalid."))
         });
 
@@ -49,21 +156,61 @@ impl Globals {
             Ok(k) => k,
             Err(e) => {
                 error!("Keypair invalid. Deleting...");
-                globals.remove("keypair")?;
+                globals.remove(b"keypair")?;
                 return Err(e);
             }
         };
 
-        Ok(Self {
+        let tls_name_override = Arc::new(RwLock::new(TlsNameMap::new()));
+        let verifier = Arc::new(MatrixServerVerifier {
+            inner: WebPKIVerifier::new(),
+            tls_name_override: tls_name_override.clone(),
+        });
+        let mut tlsconfig = rustls::ClientConfig::new();
+        tlsconfig.dangerous().set_certificate_verifier(verifier);
+        tlsconfig.root_store =
+            rustls_native_certs::load_native_certs().expect("Error loading system certificates");
+
+        let mut reqwest_client_builder = reqwest::Client::builder()
+            .connect_timeout(Duration::from_secs(30))
+            .timeout(Duration::from_secs(60 * 3))
+            .pool_max_idle_per_host(1)
+            .use_preconfigured_tls(tlsconfig);
+        if let Some(proxy) = config.proxy.to_proxy()? {
+            reqwest_client_builder = reqwest_client_builder.proxy(proxy);
+        }
+        let reqwest_client = reqwest_client_builder.build().unwrap();
+
+        let jwt_decoding_key = config
+            .jwt_secret
+            .as_ref()
+            .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static());
+
+        let s = Self {
             globals,
             config,
             keypair: Arc::new(keypair),
-            reqwest_client: reqwest::Client::new(),
-            dns_resolver: TokioAsyncResolver::tokio_from_system_conf().await.map_err(|_| {
+            reqwest_client,
+            dns_resolver: TokioAsyncResolver::tokio_from_system_conf().map_err(|_| {
                 Error::bad_config("Failed to set up trust dns resolver with system config.")
             })?,
-            actual_destination_cache: Arc::new(RwLock::new(HashMap::new())),
-        })
+            actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())),
+            tls_name_override,
+            server_signingkeys,
+            jwt_decoding_key,
+            bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())),
+            bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())),
+            servername_ratelimiter: Arc::new(RwLock::new(HashMap::new())),
+            roomid_mutex_state: RwLock::new(HashMap::new()),
+            roomid_mutex_insert: RwLock::new(HashMap::new()),
+            roomid_mutex_federation: RwLock::new(HashMap::new()),
+            sync_receivers: RwLock::new(HashMap::new()),
+            rotate: RotationHandler::new(),
+        };
+
+        fs::create_dir_all(s.get_media_folder())?;
+
+        Ok(s)
     }
 
     /// Returns this server's keypair.
@@ -76,20 +223,17 @@ impl Globals {
         &self.reqwest_client
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn next_count(&self) -> Result<u64> {
-        Ok(utils::u64_from_bytes(
-            &self
-                .globals
-                .update_and_fetch(COUNTER, utils::increment)?
-                .expect("utils::increment will always put in a value"),
-        )
-        .map_err(|_| Error::bad_database("Count has invalid bytes."))?)
+        utils::u64_from_bytes(&self.globals.increment(COUNTER)?)
+            .map_err(|_| Error::bad_database("Count has invalid bytes."))
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn current_count(&self) -> Result<u64> {
         self.globals.get(COUNTER)?.map_or(Ok(0_u64), |bytes| {
-            Ok(utils::u64_from_bytes(&bytes)
-                .map_err(|_| Error::bad_database("Count has invalid bytes."))?)
+            utils::u64_from_bytes(&bytes)
+                .map_err(|_| Error::bad_database("Count has invalid bytes."))
         })
     }
 
@@ -101,19 +245,110 @@ impl Globals {
         self.config.max_request_size
     }
 
-    pub fn registration_disabled(&self) -> bool {
-        self.config.registration_disabled
+    pub fn allow_registration(&self) -> bool {
+        self.config.allow_registration
     }
 
-    pub fn encryption_disabled(&self) -> bool {
-        self.config.encryption_disabled
+    pub fn allow_encryption(&self) -> bool {
+        self.config.allow_encryption
     }
 
-    pub fn federation_enabled(&self) -> bool {
-        self.config.federation_enabled
+    pub fn allow_federation(&self) -> bool {
+        self.config.allow_federation
+    }
+
+    pub fn trusted_servers(&self) -> &[Box<ServerName>] {
+        &self.config.trusted_servers
     }
 
     pub fn dns_resolver(&self) -> &TokioAsyncResolver {
         &self.dns_resolver
     }
+
+    pub fn jwt_decoding_key(&self) -> Option<&jsonwebtoken::DecodingKey<'_>> {
+        self.jwt_decoding_key.as_ref()
+    }
+
+    /// TODO: the key valid until timestamp is only honored in room version > 4
+    /// Remove the outdated keys and insert the new ones.
+    ///
+    /// This doesn't actually check that the keys provided are newer than the old set.
+    pub fn add_signing_key(&self, origin: &ServerName, new_keys: ServerSigningKeys) -> Result<()> {
+        // Not atomic, but this is not critical
+        let signingkeys = self.server_signingkeys.get(origin.as_bytes())?;
+
+        let mut keys = signingkeys
+            .and_then(|keys| serde_json::from_slice(&keys).ok())
+            .unwrap_or_else(|| {
+                // Just insert "now", it doesn't matter
+                ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now())
+            });
+
+        let ServerSigningKeys {
+            verify_keys,
+            old_verify_keys,
+            ..
+        } = new_keys;
+
+        keys.verify_keys.extend(verify_keys.into_iter());
+        keys.old_verify_keys.extend(old_verify_keys.into_iter());
+
+        self.server_signingkeys.insert(
+            origin.as_bytes(),
+            &serde_json::to_vec(&keys).expect("serversigningkeys can be serialized"),
+        )?;
+
+        Ok(())
+    }
+
+    /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found for the server.
+    pub fn signing_keys_for(
+        &self,
+        origin: &ServerName,
+    ) -> Result<BTreeMap<ServerSigningKeyId, VerifyKey>> {
+        let signingkeys = self
+            .server_signingkeys
+            .get(origin.as_bytes())?
+            .and_then(|bytes| serde_json::from_slice::<ServerSigningKeys>(&bytes).ok())
+            .map(|keys| {
+                let mut tree = keys.verify_keys;
+                tree.extend(
+                    keys.old_verify_keys
+                        .into_iter()
+                        .map(|old| (old.0, VerifyKey::new(old.1.key))),
+                );
+                tree
+            })
+            .unwrap_or_else(BTreeMap::new);
+
+        Ok(signingkeys)
+    }
+
+    pub fn database_version(&self) -> Result<u64> {
+        self.globals.get(b"version")?.map_or(Ok(0), |version| {
+            utils::u64_from_bytes(&version)
+                .map_err(|_| Error::bad_database("Database version id is invalid."))
+        })
+    }
+
+    pub fn bump_database_version(&self, new_version: u64) -> Result<()> {
+        self.globals
+            .insert(b"version", &new_version.to_be_bytes())?;
+        Ok(())
+    }
+
+    pub fn get_media_folder(&self) -> PathBuf {
+        let mut r = PathBuf::new();
+        r.push(self.config.database_path.clone());
+        r.push("media");
+        r
+    }
+
+    pub fn get_media_file(&self, key: &[u8]) -> PathBuf {
+        let mut r = PathBuf::new();
+        r.push(self.config.database_path.clone());
+        r.push("media");
+        r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD));
+        r
+    }
 }

src/database/key_backups.rs

@@ -2,17 +2,18 @@ use crate::{utils, Error, Result};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::backup::{BackupAlgorithm, KeyData, Sessions},
+        r0::backup::{BackupAlgorithm, KeyBackupData, RoomKeyBackup},
     },
     RoomId, UserId,
 };
-use std::{collections::BTreeMap, convert::TryFrom};
+use std::{collections::BTreeMap, convert::TryFrom, sync::Arc};
+
+use super::abstraction::Tree;
 
-#[derive(Clone)]
 pub struct KeyBackups {
-    pub(super) backupid_algorithm: sled::Tree, // BackupId = UserId + Version(Count)
-    pub(super) backupid_etag: sled::Tree,      // BackupId = UserId + Version(Count)
-    pub(super) backupkeyid_backup: sled::Tree, // BackupKeyId = UserId + Version + RoomId + SessionId
+    pub(super) backupid_algorithm: Arc<dyn Tree>, // BackupId = UserId + Version(Count)
+    pub(super) backupid_etag: Arc<dyn Tree>,      // BackupId = UserId + Version(Count)
+    pub(super) backupkeyid_backup: Arc<dyn Tree>, // BackupKeyId = UserId + Version + RoomId + SessionId
 }
 
 impl KeyBackups {
@@ -24,14 +25,13 @@ impl KeyBackups {
     ) -> Result<String> {
         let version = globals.next_count()?.to_string();
 
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
 
         self.backupid_algorithm.insert(
             &key,
-            &*serde_json::to_string(backup_metadata)
-                .expect("BackupAlgorithm::to_string always works"),
+            &serde_json::to_vec(backup_metadata).expect("BackupAlgorithm::to_vec always works"),
         )?;
         self.backupid_etag
             .insert(&key, &globals.next_count()?.to_be_bytes())?;
@@ -39,7 +39,7 @@ impl KeyBackups {
     }
 
     pub fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
 
@@ -48,13 +48,8 @@ impl KeyBackups {
 
         key.push(0xff);
 
-        for outdated_key in self
-            .backupkeyid_backup
-            .scan_prefix(&key)
-            .keys()
-            .filter_map(|r| r.ok())
-        {
-            self.backupkeyid_backup.remove(outdated_key)?;
+        for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
+            self.backupkeyid_backup.remove(&outdated_key)?;
         }
 
         Ok(())
@@ -67,7 +62,7 @@ impl KeyBackups {
         backup_metadata: &BackupAlgorithm,
         globals: &super::globals::Globals,
     ) -> Result<String> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
 
@@ -80,8 +75,9 @@ impl KeyBackups {
 
         self.backupid_algorithm.insert(
             &key,
-            &*serde_json::to_string(backup_metadata)
-                .expect("BackupAlgorithm::to_string always works"),
+            &serde_json::to_string(backup_metadata)
+                .expect("BackupAlgorithm::to_string always works")
+                .as_bytes(),
         )?;
         self.backupid_etag
             .insert(&key, &globals.next_count()?.to_be_bytes())?;
@@ -89,13 +85,16 @@ impl KeyBackups {
     }
 
     pub fn get_latest_backup(&self, user_id: &UserId) -> Result<Option<(String, BackupAlgorithm)>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
+        let mut last_possible_key = prefix.clone();
+        last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
+
         self.backupid_algorithm
-            .scan_prefix(&prefix)
-            .last()
-            .map_or(Ok(None), |r| {
-                let (key, value) = r?;
+            .iter_from(&last_possible_key, true)
+            .take_while(move |(k, _)| k.starts_with(&prefix))
+            .next()
+            .map_or(Ok(None), |(key, value)| {
                 let version = utils::string_from_bytes(
                     key.rsplit(|&b| b == 0xff)
                         .next()
@@ -113,14 +112,16 @@ impl KeyBackups {
     }
 
     pub fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Option<BackupAlgorithm>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(version.as_bytes());
 
-        self.backupid_algorithm.get(key)?.map_or(Ok(None), |bytes| {
-            Ok(serde_json::from_slice(&bytes)
-                .map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid."))?)
-        })
+        self.backupid_algorithm
+            .get(&key)?
+            .map_or(Ok(None), |bytes| {
+                serde_json::from_slice(&bytes)
+                    .map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid."))
+            })
     }
 
     pub fn add_key(
@@ -129,10 +130,10 @@ impl KeyBackups {
         version: &str,
         room_id: &RoomId,
         session_id: &str,
-        key_data: &KeyData,
+        key_data: &KeyBackupData,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(version.as_bytes());
 
@@ -147,28 +148,28 @@ impl KeyBackups {
             .insert(&key, &globals.next_count()?.to_be_bytes())?;
 
         key.push(0xff);
-        key.extend_from_slice(room_id.to_string().as_bytes());
+        key.extend_from_slice(room_id.as_bytes());
         key.push(0xff);
         key.extend_from_slice(session_id.as_bytes());
 
         self.backupkeyid_backup.insert(
             &key,
-            &*serde_json::to_string(&key_data).expect("KeyData::to_string always works"),
+            &serde_json::to_vec(&key_data).expect("KeyBackupData::to_vec always works"),
         )?;
 
         Ok(())
     }
 
     pub fn count_keys(&self, user_id: &UserId, version: &str) -> Result<usize> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
 
-        Ok(self.backupkeyid_backup.scan_prefix(&prefix).count())
+        Ok(self.backupkeyid_backup.scan_prefix(prefix).count())
     }
 
     pub fn get_etag(&self, user_id: &UserId, version: &str) -> Result<String> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
 
@@ -182,44 +183,53 @@ impl KeyBackups {
         .to_string())
     }
 
-    pub fn get_all(&self, user_id: &UserId, version: &str) -> Result<BTreeMap<RoomId, Sessions>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+    pub fn get_all(
+        &self,
+        user_id: &UserId,
+        version: &str,
+    ) -> Result<BTreeMap<RoomId, RoomKeyBackup>> {
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
         prefix.push(0xff);
 
-        let mut rooms = BTreeMap::<RoomId, Sessions>::new();
+        let mut rooms = BTreeMap::<RoomId, RoomKeyBackup>::new();
 
-        for result in self.backupkeyid_backup.scan_prefix(&prefix).map(|r| {
-            let (key, value) = r?;
-            let mut parts = key.rsplit(|&b| b == 0xff);
+        for result in self
+            .backupkeyid_backup
+            .scan_prefix(prefix)
+            .map(|(key, value)| {
+                let mut parts = key.rsplit(|&b| b == 0xff);
 
-            let session_id = utils::string_from_bytes(
-                &parts
-                    .next()
-                    .ok_or_else(|| Error::bad_database("backupkeyid_backup key is invalid."))?,
-            )
-            .map_err(|_| Error::bad_database("backupkeyid_backup session_id is invalid."))?;
+                let session_id =
+                    utils::string_from_bytes(&parts.next().ok_or_else(|| {
+                        Error::bad_database("backupkeyid_backup key is invalid.")
+                    })?)
+                    .map_err(|_| {
+                        Error::bad_database("backupkeyid_backup session_id is invalid.")
+                    })?;
 
-            let room_id = RoomId::try_from(
-                utils::string_from_bytes(
-                    &parts
-                        .next()
-                        .ok_or_else(|| Error::bad_database("backupkeyid_backup key is invalid."))?,
+                let room_id = RoomId::try_from(
+                    utils::string_from_bytes(&parts.next().ok_or_else(|| {
+                        Error::bad_database("backupkeyid_backup key is invalid.")
+                    })?)
+                    .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid."))?,
                 )
-                .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid."))?,
-            )
-            .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid room id."))?;
+                .map_err(|_| {
+                    Error::bad_database("backupkeyid_backup room_id is invalid room id.")
+                })?;
 
-            let key_data = serde_json::from_slice(&value)
-                .map_err(|_| Error::bad_database("KeyData in backupkeyid_backup is invalid."))?;
+                let key_data = serde_json::from_slice(&value).map_err(|_| {
+                    Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
+                })?;
 
-            Ok::<_, Error>((room_id, session_id, key_data))
-        }) {
+                Ok::<_, Error>((room_id, session_id, key_data))
+            })
+        {
             let (room_id, session_id, key_data) = result?;
             rooms
                 .entry(room_id)
-                .or_insert_with(|| Sessions {
+                .or_insert_with(|| RoomKeyBackup {
                     sessions: BTreeMap::new(),
                 })
                 .sessions
@@ -234,18 +244,18 @@ impl KeyBackups {
         user_id: &UserId,
         version: &str,
         room_id: &RoomId,
-    ) -> BTreeMap<String, KeyData> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+    ) -> Result<BTreeMap<String, KeyBackupData>> {
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(version.as_bytes());
         prefix.push(0xff);
         prefix.extend_from_slice(room_id.as_bytes());
         prefix.push(0xff);
 
-        self.backupkeyid_backup
-            .scan_prefix(&prefix)
-            .map(|r| {
-                let (key, value) = r?;
+        Ok(self
+            .backupkeyid_backup
+            .scan_prefix(prefix)
+            .map(|(key, value)| {
                 let mut parts = key.rsplit(|&b| b == 0xff);
 
                 let session_id =
@@ -257,13 +267,13 @@ impl KeyBackups {
                     })?;
 
                 let key_data = serde_json::from_slice(&value).map_err(|_| {
-                    Error::bad_database("KeyData in backupkeyid_backup is invalid.")
+                    Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
                 })?;
 
                 Ok::<_, Error>((session_id, key_data))
             })
             .filter_map(|r| r.ok())
-            .collect()
+            .collect())
     }
 
     pub fn get_session(
@@ -272,8 +282,8 @@ impl KeyBackups {
         version: &str,
         room_id: &RoomId,
         session_id: &str,
-    ) -> Result<Option<KeyData>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+    ) -> Result<Option<KeyBackupData>> {
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(version.as_bytes());
         key.push(0xff);
@@ -284,25 +294,21 @@ impl KeyBackups {
         self.backupkeyid_backup
             .get(&key)?
             .map(|value| {
-                serde_json::from_slice(&value)
-                    .map_err(|_| Error::bad_database("KeyData in backupkeyid_backup is invalid."))
+                serde_json::from_slice(&value).map_err(|_| {
+                    Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
+                })
             })
             .transpose()
     }
 
     pub fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
         key.push(0xff);
 
-        for outdated_key in self
-            .backupkeyid_backup
-            .scan_prefix(&key)
-            .keys()
-            .filter_map(|r| r.ok())
-        {
-            self.backupkeyid_backup.remove(outdated_key)?;
+        for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
+            self.backupkeyid_backup.remove(&outdated_key)?;
         }
 
         Ok(())
@@ -314,20 +320,15 @@ impl KeyBackups {
         version: &str,
         room_id: &RoomId,
     ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
         key.push(0xff);
         key.extend_from_slice(&room_id.as_bytes());
         key.push(0xff);
 
-        for outdated_key in self
-            .backupkeyid_backup
-            .scan_prefix(&key)
-            .keys()
-            .filter_map(|r| r.ok())
-        {
-            self.backupkeyid_backup.remove(outdated_key)?;
+        for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
+            self.backupkeyid_backup.remove(&outdated_key)?;
         }
 
         Ok(())
@@ -340,7 +341,7 @@ impl KeyBackups {
         room_id: &RoomId,
         session_id: &str,
     ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&version.as_bytes());
         key.push(0xff);
@@ -348,13 +349,8 @@ impl KeyBackups {
         key.push(0xff);
         key.extend_from_slice(&session_id.as_bytes());
 
-        for outdated_key in self
-            .backupkeyid_backup
-            .scan_prefix(&key)
-            .keys()
-            .filter_map(|r| r.ok())
-        {
-            self.backupkeyid_backup.remove(outdated_key)?;
+        for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
+            self.backupkeyid_backup.remove(&outdated_key)?;
         }
 
         Ok(())

src/database/media.rs

@@ -1,25 +1,28 @@
+use crate::database::globals::Globals;
 use image::{imageops::FilterType, GenericImageView};
 
+use super::abstraction::Tree;
 use crate::{utils, Error, Result};
-use std::mem;
+use std::{mem, sync::Arc};
+use tokio::{fs::File, io::AsyncReadExt, io::AsyncWriteExt};
 
 pub struct FileMeta {
-    pub filename: Option<String>,
+    pub content_disposition: Option<String>,
     pub content_type: Option<String>,
     pub file: Vec<u8>,
 }
 
-#[derive(Clone)]
 pub struct Media {
-    pub(super) mediaid_file: sled::Tree, // MediaId = MXC + WidthHeight + Filename + ContentType
+    pub(super) mediaid_file: Arc<dyn Tree>, // MediaId = MXC + WidthHeight + ContentDisposition + ContentType
 }
 
 impl Media {
-    /// Uploads or replaces a file.
-    pub fn create(
+    /// Uploads a file.
+    pub async fn create(
         &self,
         mxc: String,
-        filename: &Option<&str>,
+        globals: &Globals,
+        content_disposition: &Option<&str>,
         content_type: &Option<&str>,
         file: &[u8],
     ) -> Result<()> {
@@ -28,7 +31,12 @@ impl Media {
         key.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail
         key.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
         key.push(0xff);
-        key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default());
+        key.extend_from_slice(
+            content_disposition
+                .as_ref()
+                .map(|f| f.as_bytes())
+                .unwrap_or_default(),
+        );
         key.push(0xff);
         key.extend_from_slice(
             content_type
@@ -37,16 +45,21 @@ impl Media {
                 .unwrap_or_default(),
         );
 
-        self.mediaid_file.insert(key, file)?;
+        let path = globals.get_media_file(&key);
+        let mut f = File::create(path).await?;
+        f.write_all(file).await?;
 
+        self.mediaid_file.insert(&key, &[])?;
         Ok(())
     }
 
     /// Uploads or replaces a file thumbnail.
-    pub fn upload_thumbnail(
+    #[allow(clippy::too_many_arguments)]
+    pub async fn upload_thumbnail(
         &self,
         mxc: String,
-        filename: &Option<String>,
+        globals: &Globals,
+        content_disposition: &Option<String>,
         content_type: &Option<String>,
         width: u32,
         height: u32,
@@ -57,7 +70,12 @@ impl Media {
         key.extend_from_slice(&width.to_be_bytes());
         key.extend_from_slice(&height.to_be_bytes());
         key.push(0xff);
-        key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default());
+        key.extend_from_slice(
+            content_disposition
+                .as_ref()
+                .map(|f| f.as_bytes())
+                .unwrap_or_default(),
+        );
         key.push(0xff);
         key.extend_from_slice(
             content_type
@@ -66,48 +84,59 @@ impl Media {
                 .unwrap_or_default(),
         );
 
-        self.mediaid_file.insert(key, file)?;
+        let path = globals.get_media_file(&key);
+        let mut f = File::create(path).await?;
+        f.write_all(file).await?;
+
+        self.mediaid_file.insert(&key, &[])?;
 
         Ok(())
     }
 
     /// Downloads a file.
-    pub fn get(&self, mxc: &str) -> Result<Option<FileMeta>> {
+    pub async fn get(&self, globals: &Globals, mxc: &str) -> Result<Option<FileMeta>> {
         let mut prefix = mxc.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail
         prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
         prefix.push(0xff);
 
-        if let Some(r) = self.mediaid_file.scan_prefix(&prefix).next() {
-            let (key, file) = r?;
+        let first = self.mediaid_file.scan_prefix(prefix).next();
+        if let Some((key, _)) = first {
+            let path = globals.get_media_file(&key);
+            let mut file = Vec::new();
+            File::open(path).await?.read_to_end(&mut file).await?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
                 .next()
                 .map(|bytes| {
-                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
+                    utils::string_from_bytes(bytes).map_err(|_| {
                         Error::bad_database("Content type in mediaid_file is invalid unicode.")
-                    })?)
+                    })
                 })
                 .transpose()?;
 
-            let filename_bytes = parts
+            let content_disposition_bytes = parts
                 .next()
                 .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?;
 
-            let filename = if filename_bytes.is_empty() {
+            let content_disposition = if content_disposition_bytes.is_empty() {
                 None
             } else {
-                Some(utils::string_from_bytes(filename_bytes).map_err(|_| {
-                    Error::bad_database("Filename in mediaid_file is invalid unicode.")
-                })?)
+                Some(
+                    utils::string_from_bytes(content_disposition_bytes).map_err(|_| {
+                        Error::bad_database(
+                            "Content Disposition in mediaid_file is invalid unicode.",
+                        )
+                    })?,
+                )
             };
 
             Ok(Some(FileMeta {
-                filename,
+                content_disposition,
                 content_type,
-                file: file.to_vec(),
+                file,
             }))
         } else {
             Ok(None)
@@ -137,7 +166,13 @@ impl Media {
     /// - Server creates the thumbnail and sends it to the user
     ///
     /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
-    pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result<Option<FileMeta>> {
+    pub async fn get_thumbnail(
+        &self,
+        mxc: String,
+        globals: &Globals,
+        width: u32,
+        height: u32,
+    ) -> Result<Option<FileMeta>> {
         let (width, height, crop) = self
             .thumbnail_properties(width, height)
             .unwrap_or((0, 0, false)); // 0, 0 because that's the original file
@@ -155,63 +190,74 @@ impl Media {
         original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
         original_prefix.push(0xff);
 
-        if let Some(r) = self.mediaid_file.scan_prefix(&thumbnail_prefix).next() {
+        let first_thumbnailprefix = self.mediaid_file.scan_prefix(thumbnail_prefix).next();
+        let first_originalprefix = self.mediaid_file.scan_prefix(original_prefix).next();
+        if let Some((key, _)) = first_thumbnailprefix {
             // Using saved thumbnail
-            let (key, file) = r?;
+            let path = globals.get_media_file(&key);
+            let mut file = Vec::new();
+            File::open(path).await?.read_to_end(&mut file).await?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
                 .next()
                 .map(|bytes| {
-                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
+                    utils::string_from_bytes(bytes).map_err(|_| {
                         Error::bad_database("Content type in mediaid_file is invalid unicode.")
-                    })?)
+                    })
                 })
                 .transpose()?;
 
-            let filename_bytes = parts
+            let content_disposition_bytes = parts
                 .next()
                 .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?;
 
-            let filename = if filename_bytes.is_empty() {
+            let content_disposition = if content_disposition_bytes.is_empty() {
                 None
             } else {
                 Some(
-                    utils::string_from_bytes(filename_bytes)
-                        .map_err(|_| Error::bad_database("Filename in db is invalid."))?,
+                    utils::string_from_bytes(content_disposition_bytes).map_err(|_| {
+                        Error::bad_database("Content Disposition in db is invalid.")
+                    })?,
                 )
             };
 
             Ok(Some(FileMeta {
-                filename,
+                content_disposition,
                 content_type,
                 file: file.to_vec(),
             }))
-        } else if let Some(r) = self.mediaid_file.scan_prefix(&original_prefix).next() {
+        } else if let Some((key, _)) = first_originalprefix {
             // Generate a thumbnail
+            let path = globals.get_media_file(&key);
+            let mut file = Vec::new();
+            File::open(path).await?.read_to_end(&mut file).await?;
 
-            let (key, file) = r?;
             let mut parts = key.rsplit(|&b| b == 0xff);
 
             let content_type = parts
                 .next()
                 .map(|bytes| {
-                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
+                    utils::string_from_bytes(bytes).map_err(|_| {
                         Error::bad_database("Content type in mediaid_file is invalid unicode.")
-                    })?)
+                    })
                 })
                 .transpose()?;
 
-            let filename_bytes = parts
+            let content_disposition_bytes = parts
                 .next()
                 .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?;
 
-            let filename = if filename_bytes.is_empty() {
+            let content_disposition = if content_disposition_bytes.is_empty() {
                 None
             } else {
-                Some(utils::string_from_bytes(filename_bytes).map_err(|_| {
-                    Error::bad_database("Filename in mediaid_file is invalid unicode.")
-                })?)
+                Some(
+                    utils::string_from_bytes(content_disposition_bytes).map_err(|_| {
+                        Error::bad_database(
+                            "Content Disposition in mediaid_file is invalid unicode.",
+                        )
+                    })?,
+                )
             };
 
             if let Ok(image) = image::load_from_memory(&file) {
@@ -219,23 +265,24 @@ impl Media {
                 let original_height = image.height();
                 if width > original_width || height > original_height {
                     return Ok(Some(FileMeta {
-                        filename,
+                        content_disposition,
                         content_type,
                         file: file.to_vec(),
                     }));
                 }
 
                 let thumbnail = if crop {
-                    image.resize_to_fill(width, height, FilterType::Triangle)
+                    image.resize_to_fill(width, height, FilterType::CatmullRom)
                 } else {
                     let (exact_width, exact_height) = {
                         // Copied from image::dynimage::resize_dimensions
                         let ratio = u64::from(original_width) * u64::from(height);
                         let nratio = u64::from(width) * u64::from(original_height);
 
-                        let use_width = nratio > ratio;
+                        let use_width = nratio <= ratio;
                         let intermediate = if use_width {
-                            u64::from(original_height) * u64::from(width) / u64::from(width)
+                            u64::from(original_height) * u64::from(width)
+                                / u64::from(original_width)
                         } else {
                             u64::from(original_width) * u64::from(height)
                                 / u64::from(original_height)
@@ -282,15 +329,24 @@ impl Media {
                     widthheight,
                 );
 
-                self.mediaid_file.insert(thumbnail_key, &*thumbnail_bytes)?;
+                let path = globals.get_media_file(&thumbnail_key);
+                let mut f = File::create(path).await?;
+                f.write_all(&thumbnail_bytes).await?;
+
+                self.mediaid_file.insert(&thumbnail_key, &[])?;
 
                 Ok(Some(FileMeta {
-                    filename,
+                    content_disposition,
                     content_type,
                     file: thumbnail_bytes.to_vec(),
                 }))
             } else {
-                Ok(None)
+                // Couldn't parse file to generate thumbnail, send original
+                Ok(Some(FileMeta {
+                    content_disposition,
+                    content_type,
+                    file: file.to_vec(),
+                }))
             }
         } else {
             Ok(None)

src/database/proxy.rs

@@ -0,0 +1,146 @@
+use reqwest::{Proxy, Url};
+use serde::Deserialize;
+
+use crate::Result;
+
+/// ## Examples:
+/// - No proxy (default):
+/// ```toml
+/// proxy ="none"
+/// ```
+/// - Global proxy
+/// ```toml
+/// [proxy]
+/// global = { url = "socks5h://localhost:9050" }
+/// ```
+/// - Proxy some domains
+/// ```toml
+/// [proxy]
+/// [[proxy.by_domain]]
+/// url = "socks5h://localhost:9050"
+/// include = ["*.onion", "matrix.myspecial.onion"]
+/// exclude = ["*.myspecial.onion"]
+/// ```
+/// ## Include vs. Exclude
+/// If include is an empty list, it is assumed to be `["*"]`.
+///
+/// If a domain matches both the exclude and include list, the proxy will only be used if it was
+/// included because of a more specific rule than it was excluded. In the above example, the proxy
+/// would be used for `ordinary.onion`, `matrix.myspecial.onion`, but not `hello.myspecial.onion`.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProxyConfig {
+    None,
+    Global {
+        #[serde(deserialize_with = "crate::utils::deserialize_from_str")]
+        url: Url,
+    },
+    ByDomain(Vec<PartialProxyConfig>),
+}
+impl ProxyConfig {
+    pub fn to_proxy(&self) -> Result<Option<Proxy>> {
+        Ok(match self.clone() {
+            ProxyConfig::None => None,
+            ProxyConfig::Global { url } => Some(Proxy::all(url)?),
+            ProxyConfig::ByDomain(proxies) => Some(Proxy::custom(move |url| {
+                proxies.iter().find_map(|proxy| proxy.for_url(url)).cloned() // first matching proxy
+            })),
+        })
+    }
+}
+impl Default for ProxyConfig {
+    fn default() -> Self {
+        ProxyConfig::None
+    }
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct PartialProxyConfig {
+    #[serde(deserialize_with = "crate::utils::deserialize_from_str")]
+    url: Url,
+    #[serde(default)]
+    include: Vec<WildCardedDomain>,
+    #[serde(default)]
+    exclude: Vec<WildCardedDomain>,
+}
+impl PartialProxyConfig {
+    pub fn for_url(&self, url: &Url) -> Option<&Url> {
+        let domain = url.domain()?;
+        let mut included_because = None; // most specific reason it was included
+        let mut excluded_because = None; // most specific reason it was excluded
+        if self.include.is_empty() {
+            // treat empty include list as `*`
+            included_because = Some(&WildCardedDomain::WildCard)
+        }
+        for wc_domain in &self.include {
+            if wc_domain.matches(domain) {
+                match included_because {
+                    Some(prev) if !wc_domain.more_specific_than(prev) => (),
+                    _ => included_because = Some(wc_domain),
+                }
+            }
+        }
+        for wc_domain in &self.exclude {
+            if wc_domain.matches(domain) {
+                match excluded_because {
+                    Some(prev) if !wc_domain.more_specific_than(prev) => (),
+                    _ => excluded_because = Some(wc_domain),
+                }
+            }
+        }
+        match (included_because, excluded_because) {
+            (Some(a), Some(b)) if a.more_specific_than(b) => Some(&self.url), // included for a more specific reason than excluded
+            (Some(_), None) => Some(&self.url),
+            _ => None,
+        }
+    }
+}
+
+/// A domain name, that optionally allows a * as its first subdomain.
+#[derive(Clone, Debug)]
+pub enum WildCardedDomain {
+    WildCard,
+    WildCarded(String),
+    Exact(String),
+}
+impl WildCardedDomain {
+    pub fn matches(&self, domain: &str) -> bool {
+        match self {
+            WildCardedDomain::WildCard => true,
+            WildCardedDomain::WildCarded(d) => domain.ends_with(d),
+            WildCardedDomain::Exact(d) => domain == d,
+        }
+    }
+    pub fn more_specific_than(&self, other: &Self) -> bool {
+        match (self, other) {
+            (WildCardedDomain::WildCard, WildCardedDomain::WildCard) => false,
+            (_, WildCardedDomain::WildCard) => true,
+            (WildCardedDomain::Exact(a), WildCardedDomain::WildCarded(_)) => other.matches(a),
+            (WildCardedDomain::WildCarded(a), WildCardedDomain::WildCarded(b)) => {
+                a != b && a.ends_with(b)
+            }
+            _ => false,
+        }
+    }
+}
+impl std::str::FromStr for WildCardedDomain {
+    type Err = std::convert::Infallible;
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        // maybe do some domain validation?
+        Ok(if s.starts_with("*.") {
+            WildCardedDomain::WildCarded(s[1..].to_owned())
+        } else if s == "*" {
+            WildCardedDomain::WildCarded("".to_owned())
+        } else {
+            WildCardedDomain::Exact(s.to_owned())
+        })
+    }
+}
+impl<'de> serde::de::Deserialize<'de> for WildCardedDomain {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'de>,
+    {
+        crate::utils::deserialize_from_str(deserializer)
+    }
+}

src/database/pusher.rs

@@ -0,0 +1,340 @@
+use crate::{Database, Error, PduEvent, Result};
+use bytes::BytesMut;
+use ruma::{
+    api::{
+        client::r0::push::{get_pushers, set_pusher, PusherKind},
+        push_gateway::send_event_notification::{
+            self,
+            v1::{Device, Notification, NotificationCounts, NotificationPriority},
+        },
+        IncomingResponse, OutgoingRequest, SendAccessToken,
+    },
+    events::{room::power_levels::PowerLevelsEventContent, AnySyncRoomEvent, EventType},
+    identifiers::RoomName,
+    push::{Action, PushConditionRoomCtx, PushFormat, Ruleset, Tweak},
+    serde::Raw,
+    uint, RoomId, UInt, UserId,
+};
+use tracing::{error, info, warn};
+
+use std::{convert::TryFrom, fmt::Debug, mem, sync::Arc};
+
+use super::abstraction::Tree;
+
+pub struct PushData {
+    /// UserId + pushkey -> Pusher
+    pub(super) senderkey_pusher: Arc<dyn Tree>,
+}
+
+impl PushData {
+    #[tracing::instrument(skip(self, sender, pusher))]
+    pub fn set_pusher(&self, sender: &UserId, pusher: set_pusher::Pusher) -> Result<()> {
+        let mut key = sender.as_bytes().to_vec();
+        key.push(0xff);
+        key.extend_from_slice(pusher.pushkey.as_bytes());
+
+        // There are 2 kinds of pushers but the spec says: null deletes the pusher.
+        if pusher.kind.is_none() {
+            return self
+                .senderkey_pusher
+                .remove(&key)
+                .map(|_| ())
+                .map_err(Into::into);
+        }
+
+        self.senderkey_pusher.insert(
+            &key,
+            &serde_json::to_vec(&pusher).expect("Pusher is valid JSON value"),
+        )?;
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, senderkey))]
+    pub fn get_pusher(&self, senderkey: &[u8]) -> Result<Option<get_pushers::Pusher>> {
+        self.senderkey_pusher
+            .get(senderkey)?
+            .map(|push| {
+                serde_json::from_slice(&*push)
+                    .map_err(|_| Error::bad_database("Invalid Pusher in db."))
+            })
+            .transpose()
+    }
+
+    #[tracing::instrument(skip(self, sender))]
+    pub fn get_pushers(&self, sender: &UserId) -> Result<Vec<get_pushers::Pusher>> {
+        let mut prefix = sender.as_bytes().to_vec();
+        prefix.push(0xff);
+
+        self.senderkey_pusher
+            .scan_prefix(prefix)
+            .map(|(_, push)| {
+                serde_json::from_slice(&*push)
+                    .map_err(|_| Error::bad_database("Invalid Pusher in db."))
+            })
+            .collect()
+    }
+
+    #[tracing::instrument(skip(self, sender))]
+    pub fn get_pusher_senderkeys<'a>(
+        &'a self,
+        sender: &UserId,
+    ) -> impl Iterator<Item = Vec<u8>> + 'a {
+        let mut prefix = sender.as_bytes().to_vec();
+        prefix.push(0xff);
+
+        self.senderkey_pusher.scan_prefix(prefix).map(|(k, _)| k)
+    }
+}
+
+#[tracing::instrument(skip(globals, destination, request))]
+pub async fn send_request<T: OutgoingRequest>(
+    globals: &crate::database::globals::Globals,
+    destination: &str,
+    request: T,
+) -> Result<T::IncomingResponse>
+where
+    T: Debug,
+{
+    let destination = destination.replace("/_matrix/push/v1/notify", "");
+
+    let http_request = request
+        .try_into_http_request::<BytesMut>(&destination, SendAccessToken::IfRequired(""))
+        .map_err(|e| {
+            warn!("Failed to find destination {}: {}", destination, e);
+            Error::BadServerResponse("Invalid destination")
+        })?
+        .map(|body| body.freeze());
+
+    let reqwest_request = reqwest::Request::try_from(http_request)
+        .expect("all http requests are valid reqwest requests");
+
+    // TODO: we could keep this very short and let expo backoff do it's thing...
+    //*reqwest_request.timeout_mut() = Some(Duration::from_secs(5));
+
+    let url = reqwest_request.url().clone();
+    let response = globals.reqwest_client().execute(reqwest_request).await;
+
+    match response {
+        Ok(mut response) => {
+            // reqwest::Response -> http::Response conversion
+            let status = response.status();
+            let mut http_response_builder = http::Response::builder()
+                .status(status)
+                .version(response.version());
+            mem::swap(
+                response.headers_mut(),
+                http_response_builder
+                    .headers_mut()
+                    .expect("http::response::Builder is usable"),
+            );
+
+            let body = response.bytes().await.unwrap_or_else(|e| {
+                warn!("server error {}", e);
+                Vec::new().into()
+            }); // TODO: handle timeout
+
+            if status != 200 {
+                info!(
+                    "Push gateway returned bad response {} {}\n{}\n{:?}",
+                    destination,
+                    status,
+                    url,
+                    crate::utils::string_from_bytes(&body)
+                );
+            }
+
+            let response = T::IncomingResponse::try_from_http_response(
+                http_response_builder
+                    .body(body)
+                    .expect("reqwest body is valid http body"),
+            );
+            response.map_err(|_| {
+                info!(
+                    "Push gateway returned invalid response bytes {}\n{}",
+                    destination, url
+                );
+                Error::BadServerResponse("Push gateway returned bad response.")
+            })
+        }
+        Err(e) => Err(e.into()),
+    }
+}
+
+#[tracing::instrument(skip(user, unread, pusher, ruleset, pdu, db))]
+pub async fn send_push_notice(
+    user: &UserId,
+    unread: UInt,
+    pusher: &get_pushers::Pusher,
+    ruleset: Ruleset,
+    pdu: &PduEvent,
+    db: &Database,
+) -> Result<()> {
+    let mut notify = None;
+    let mut tweaks = Vec::new();
+
+    let power_levels: PowerLevelsEventContent = db
+        .rooms
+        .room_state_get(&pdu.room_id, &EventType::RoomPowerLevels, "")?
+        .map(|ev| {
+            serde_json::from_value(ev.content.clone())
+                .map_err(|_| Error::bad_database("invalid m.room.power_levels event"))
+        })
+        .transpose()?
+        .unwrap_or_default();
+
+    for action in get_actions(
+        user,
+        &ruleset,
+        &power_levels,
+        &pdu.to_sync_room_event(),
+        &pdu.room_id,
+        db,
+    )? {
+        let n = match action {
+            Action::DontNotify => false,
+            // TODO: Implement proper support for coalesce
+            Action::Notify | Action::Coalesce => true,
+            Action::SetTweak(tweak) => {
+                tweaks.push(tweak.clone());
+                continue;
+            }
+        };
+
+        if notify.is_some() {
+            return Err(Error::bad_database(
+                r#"Malformed pushrule contains more than one of these actions: ["dont_notify", "notify", "coalesce"]"#,
+            ));
+        }
+
+        notify = Some(n);
+    }
+
+    if notify == Some(true) {
+        send_notice(unread, pusher, tweaks, pdu, db).await?;
+    }
+    // Else the event triggered no actions
+
+    Ok(())
+}
+
+#[tracing::instrument(skip(user, ruleset, pdu, db))]
+pub fn get_actions<'a>(
+    user: &UserId,
+    ruleset: &'a Ruleset,
+    power_levels: &PowerLevelsEventContent,
+    pdu: &Raw<AnySyncRoomEvent>,
+    room_id: &RoomId,
+    db: &Database,
+) -> Result<&'a [Action]> {
+    let ctx = PushConditionRoomCtx {
+        room_id: room_id.clone(),
+        member_count: 10_u32.into(), // TODO: get member count efficiently
+        user_display_name: db
+            .users
+            .displayname(&user)?
+            .unwrap_or_else(|| user.localpart().to_owned()),
+        users_power_levels: power_levels.users.clone(),
+        default_power_level: power_levels.users_default,
+        notification_power_levels: power_levels.notifications.clone(),
+    };
+
+    Ok(ruleset.get_actions(pdu, &ctx))
+}
+
+#[tracing::instrument(skip(unread, pusher, tweaks, event, db))]
+async fn send_notice(
+    unread: UInt,
+    pusher: &get_pushers::Pusher,
+    tweaks: Vec<Tweak>,
+    event: &PduEvent,
+    db: &Database,
+) -> Result<()> {
+    // TODO: email
+    if pusher.kind == PusherKind::Email {
+        return Ok(());
+    }
+
+    // TODO:
+    // Two problems with this
+    // 1. if "event_id_only" is the only format kind it seems we should never add more info
+    // 2. can pusher/devices have conflicting formats
+    let event_id_only = pusher.data.format == Some(PushFormat::EventIdOnly);
+    let url = if let Some(url) = &pusher.data.url {
+        url
+    } else {
+        error!("Http Pusher must have URL specified.");
+        return Ok(());
+    };
+
+    let mut device = Device::new(pusher.app_id.clone(), pusher.pushkey.clone());
+    let mut data_minus_url = pusher.data.clone();
+    // The url must be stripped off according to spec
+    data_minus_url.url = None;
+    device.data = Some(data_minus_url);
+
+    // Tweaks are only added if the format is NOT event_id_only
+    if !event_id_only {
+        device.tweaks = tweaks.clone();
+    }
+
+    let d = &[device];
+    let mut notifi = Notification::new(d);
+
+    notifi.prio = NotificationPriority::Low;
+    notifi.event_id = Some(&event.event_id);
+    notifi.room_id = Some(&event.room_id);
+    // TODO: missed calls
+    notifi.counts = NotificationCounts::new(unread, uint!(0));
+
+    if event.kind == EventType::RoomEncrypted
+        || tweaks
+            .iter()
+            .any(|t| matches!(t, Tweak::Highlight(true) | Tweak::Sound(_)))
+    {
+        notifi.prio = NotificationPriority::High
+    }
+
+    if event_id_only {
+        send_request(
+            &db.globals,
+            &url,
+            send_event_notification::v1::Request::new(notifi),
+        )
+        .await?;
+    } else {
+        notifi.sender = Some(&event.sender);
+        notifi.event_type = Some(&event.kind);
+        let content = serde_json::value::to_raw_value(&event.content).ok();
+        notifi.content = content.as_deref();
+
+        if event.kind == EventType::RoomMember {
+            notifi.user_is_target = event.state_key.as_deref() == Some(event.sender.as_str());
+        }
+
+        let user_name = db.users.displayname(&event.sender)?;
+        notifi.sender_display_name = user_name.as_deref();
+        let room_name = db
+            .rooms
+            .room_state_get(&event.room_id, &EventType::RoomName, "")?
+            .map(|pdu| match pdu.content.get("name") {
+                Some(serde_json::Value::String(s)) => {
+                    Some(Box::<RoomName>::try_from(&**s).expect("room name is valid"))
+                }
+                _ => None,
+            })
+            .flatten();
+        notifi.room_name = room_name.as_deref();
+
+        send_request(
+            &db.globals,
+            &url,
+            send_event_notification::v1::Request::new(notifi),
+        )
+        .await?;
+    }
+
+    // TODO: email
+
+    Ok(())
+}

src/database/rooms/edus.rs

@@ -1,29 +1,29 @@
-use crate::{utils, Error, Result};
-use js_int::UInt;
+use crate::{database::abstraction::Tree, utils, Error, Result};
 use ruma::{
     events::{
         presence::{PresenceEvent, PresenceEventContent},
-        AnyEvent as EduEvent, SyncEphemeralRoomEvent,
+        AnyEphemeralRoomEvent, SyncEphemeralRoomEvent,
     },
     presence::PresenceState,
     serde::Raw,
-    RoomId, UserId,
+    signatures::CanonicalJsonObject,
+    RoomId, UInt, UserId,
 };
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
     convert::{TryFrom, TryInto},
     mem,
+    sync::Arc,
 };
 
-#[derive(Clone)]
 pub struct RoomEdus {
-    pub(in super::super) readreceiptid_readreceipt: sled::Tree, // ReadReceiptId = RoomId + Count + UserId
-    pub(in super::super) roomuserid_privateread: sled::Tree, // RoomUserId = Room + User, PrivateRead = Count
-    pub(in super::super) roomuserid_lastprivatereadupdate: sled::Tree, // LastPrivateReadUpdate = Count
-    pub(in super::super) typingid_userid: sled::Tree, // TypingId = RoomId + TimeoutTime + Count
-    pub(in super::super) roomid_lasttypingupdate: sled::Tree, // LastRoomTypingUpdate = Count
-    pub(in super::super) presenceid_presence: sled::Tree, // PresenceId = RoomId + Count + UserId
-    pub(in super::super) userid_lastpresenceupdate: sled::Tree, // LastPresenceUpdate = Count
+    pub(in super::super) readreceiptid_readreceipt: Arc<dyn Tree>, // ReadReceiptId = RoomId + Count + UserId
+    pub(in super::super) roomuserid_privateread: Arc<dyn Tree>, // RoomUserId = Room + User, PrivateRead = Count
+    pub(in super::super) roomuserid_lastprivatereadupdate: Arc<dyn Tree>, // LastPrivateReadUpdate = Count
+    pub(in super::super) typingid_userid: Arc<dyn Tree>, // TypingId = RoomId + TimeoutTime + Count
+    pub(in super::super) roomid_lasttypingupdate: Arc<dyn Tree>, // LastRoomTypingUpdate = Count
+    pub(in super::super) presenceid_presence: Arc<dyn Tree>, // PresenceId = RoomId + Count + UserId
+    pub(in super::super) userid_lastpresenceupdate: Arc<dyn Tree>, // LastPresenceUpdate = Count
 }
 
 impl RoomEdus {
@@ -32,69 +32,91 @@ impl RoomEdus {
         &self,
         user_id: &UserId,
         room_id: &RoomId,
-        event: EduEvent,
+        event: AnyEphemeralRoomEvent,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
+        let mut last_possible_key = prefix.clone();
+        last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
+
         // Remove old entry
-        if let Some(old) = self
+        if let Some((old, _)) = self
             .readreceiptid_readreceipt
-            .scan_prefix(&prefix)
-            .keys()
-            .rev()
-            .filter_map(|r| r.ok())
-            .take_while(|key| key.starts_with(&prefix))
-            .find(|key| {
+            .iter_from(&last_possible_key, true)
+            .take_while(|(key, _)| key.starts_with(&prefix))
+            .find(|(key, _)| {
                 key.rsplit(|&b| b == 0xff)
                     .next()
                     .expect("rsplit always returns an element")
-                    == user_id.to_string().as_bytes()
+                    == user_id.as_bytes()
             })
         {
             // This is the old room_latest
-            self.readreceiptid_readreceipt.remove(old)?;
+            self.readreceiptid_readreceipt.remove(&old)?;
         }
 
         let mut room_latest_id = prefix;
         room_latest_id.extend_from_slice(&globals.next_count()?.to_be_bytes());
         room_latest_id.push(0xff);
-        room_latest_id.extend_from_slice(&user_id.to_string().as_bytes());
+        room_latest_id.extend_from_slice(&user_id.as_bytes());
 
         self.readreceiptid_readreceipt.insert(
-            room_latest_id,
-            &*serde_json::to_string(&event).expect("EduEvent::to_string always works"),
+            &room_latest_id,
+            &serde_json::to_vec(&event).expect("EduEvent::to_string always works"),
         )?;
 
         Ok(())
     }
 
     /// Returns an iterator over the most recent read_receipts in a room that happened after the event with id `since`.
-    pub fn readreceipts_since(
-        &self,
+    #[tracing::instrument(skip(self))]
+    pub fn readreceipts_since<'a>(
+        &'a self,
         room_id: &RoomId,
         since: u64,
-    ) -> Result<impl Iterator<Item = Result<Raw<ruma::events::AnySyncEphemeralRoomEvent>>>> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+    ) -> impl Iterator<Item = Result<(UserId, u64, Raw<ruma::events::AnySyncEphemeralRoomEvent>)>> + 'a
+    {
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
+        let prefix2 = prefix.clone();
 
         let mut first_possible_edu = prefix.clone();
         first_possible_edu.extend_from_slice(&(since + 1).to_be_bytes()); // +1 so we don't send the event at since
 
-        Ok(self
-            .readreceiptid_readreceipt
-            .range(&*first_possible_edu..)
-            .filter_map(|r| r.ok())
-            .take_while(move |(k, _)| k.starts_with(&prefix))
-            .map(|(_, v)| {
-                Ok(serde_json::from_slice(&v).map_err(|_| {
-                    Error::bad_database("Read receipt in roomlatestid_roomlatest is invalid.")
-                })?)
-            }))
+        self.readreceiptid_readreceipt
+            .iter_from(&first_possible_edu, false)
+            .take_while(move |(k, _)| k.starts_with(&prefix2))
+            .map(move |(k, v)| {
+                let count =
+                    utils::u64_from_bytes(&k[prefix.len()..prefix.len() + mem::size_of::<u64>()])
+                        .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?;
+                let user_id = UserId::try_from(
+                    utils::string_from_bytes(&k[prefix.len() + mem::size_of::<u64>() + 1..])
+                        .map_err(|_| {
+                            Error::bad_database("Invalid readreceiptid userid bytes in db.")
+                        })?,
+                )
+                .map_err(|_| Error::bad_database("Invalid readreceiptid userid in db."))?;
+
+                let mut json = serde_json::from_slice::<CanonicalJsonObject>(&v).map_err(|_| {
+                    Error::bad_database("Read receipt in roomlatestid_roomlatest is invalid json.")
+                })?;
+                json.remove("room_id");
+
+                Ok((
+                    user_id,
+                    count,
+                    Raw::from_json(
+                        serde_json::value::to_raw_value(&json).expect("json is valid raw value"),
+                    ),
+                ))
+            })
     }
 
     /// Sets a private read marker at `count`.
+    #[tracing::instrument(skip(self, globals))]
     pub fn private_read_set(
         &self,
         room_id: &RoomId,
@@ -102,9 +124,9 @@ impl RoomEdus {
         count: u64,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(&user_id.as_bytes());
 
         self.roomuserid_privateread
             .insert(&key, &count.to_be_bytes())?;
@@ -116,23 +138,26 @@ impl RoomEdus {
     }
 
     /// Returns the private read marker.
+    #[tracing::instrument(skip(self))]
     pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result<Option<u64>> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(&user_id.as_bytes());
 
-        self.roomuserid_privateread.get(key)?.map_or(Ok(None), |v| {
-            Ok(Some(utils::u64_from_bytes(&v).map_err(|_| {
-                Error::bad_database("Invalid private read marker bytes")
-            })?))
-        })
+        self.roomuserid_privateread
+            .get(&key)?
+            .map_or(Ok(None), |v| {
+                Ok(Some(utils::u64_from_bytes(&v).map_err(|_| {
+                    Error::bad_database("Invalid private read marker bytes")
+                })?))
+            })
     }
 
     /// Returns the count of the last typing update in this room.
     pub fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result<u64> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(&user_id.as_bytes());
 
         Ok(self
             .roomuserid_lastprivatereadupdate
@@ -154,7 +179,7 @@ impl RoomEdus {
         timeout: u64,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let count = globals.next_count()?.to_be_bytes();
@@ -165,10 +190,10 @@ impl RoomEdus {
         room_typing_id.extend_from_slice(&count);
 
         self.typingid_userid
-            .insert(&room_typing_id, &*user_id.to_string().as_bytes())?;
+            .insert(&room_typing_id, &*user_id.as_bytes())?;
 
         self.roomid_lasttypingupdate
-            .insert(&room_id.to_string().as_bytes(), &count)?;
+            .insert(&room_id.as_bytes(), &count)?;
 
         Ok(())
     }
@@ -180,7 +205,7 @@ impl RoomEdus {
         room_id: &RoomId,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let user_id = user_id.to_string();
@@ -190,19 +215,16 @@ impl RoomEdus {
         // Maybe there are multiple ones from calling roomtyping_add multiple times
         for outdated_edu in self
             .typingid_userid
-            .scan_prefix(&prefix)
-            .filter_map(|r| r.ok())
-            .filter(|(_, v)| v == user_id.as_bytes())
+            .scan_prefix(prefix)
+            .filter(|(_, v)| &**v == user_id.as_bytes())
         {
-            self.typingid_userid.remove(outdated_edu.0)?;
+            self.typingid_userid.remove(&outdated_edu.0)?;
             found_outdated = true;
         }
 
         if found_outdated {
-            self.roomid_lasttypingupdate.insert(
-                &room_id.to_string().as_bytes(),
-                &globals.next_count()?.to_be_bytes(),
-            )?;
+            self.roomid_lasttypingupdate
+                .insert(&room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
         }
 
         Ok(())
@@ -214,7 +236,7 @@ impl RoomEdus {
         room_id: &RoomId,
         globals: &super::super::globals::Globals,
     ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let current_timestamp = utils::millis_since_unix_epoch();
@@ -224,10 +246,8 @@ impl RoomEdus {
         // Find all outdated edus before inserting a new one
         for outdated_edu in self
             .typingid_userid
-            .scan_prefix(&prefix)
-            .keys()
-            .map(|key| {
-                let key = key?;
+            .scan_prefix(prefix)
+            .map(|(key, _)| {
                 Ok::<_, Error>((
                     key.clone(),
                     utils::u64_from_bytes(
@@ -242,21 +262,20 @@ impl RoomEdus {
             .take_while(|&(_, timestamp)| timestamp < current_timestamp)
         {
             // This is an outdated edu (time > timestamp)
-            self.typingid_userid.remove(outdated_edu.0)?;
+            self.typingid_userid.remove(&outdated_edu.0)?;
             found_outdated = true;
         }
 
         if found_outdated {
-            self.roomid_lasttypingupdate.insert(
-                &room_id.to_string().as_bytes(),
-                &globals.next_count()?.to_be_bytes(),
-            )?;
+            self.roomid_lasttypingupdate
+                .insert(&room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
         }
 
         Ok(())
     }
 
     /// Returns the count of the last typing update in this room.
+    #[tracing::instrument(skip(self, globals))]
     pub fn last_typing_update(
         &self,
         room_id: &RoomId,
@@ -266,7 +285,7 @@ impl RoomEdus {
 
         Ok(self
             .roomid_lasttypingupdate
-            .get(&room_id.to_string().as_bytes())?
+            .get(&room_id.as_bytes())?
             .map_or(Ok::<_, Error>(None), |bytes| {
                 Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| {
                     Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")
@@ -279,29 +298,28 @@ impl RoomEdus {
         &self,
         room_id: &RoomId,
     ) -> Result<SyncEphemeralRoomEvent<ruma::events::typing::TypingEventContent>> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
-        let mut user_ids = Vec::new();
+        let mut user_ids = HashSet::new();
 
         for user_id in self
             .typingid_userid
             .scan_prefix(prefix)
-            .values()
-            .map(|user_id| {
-                Ok::<_, Error>(
-                    UserId::try_from(utils::string_from_bytes(&user_id?).map_err(|_| {
-                        Error::bad_database("User ID in typingid_userid is invalid unicode.")
-                    })?)
-                    .map_err(|_| Error::bad_database("User ID in typingid_userid is invalid."))?,
-                )
+            .map(|(_, user_id)| {
+                UserId::try_from(utils::string_from_bytes(&user_id).map_err(|_| {
+                    Error::bad_database("User ID in typingid_userid is invalid unicode.")
+                })?)
+                .map_err(|_| Error::bad_database("User ID in typingid_userid is invalid."))
             })
         {
-            user_ids.push(user_id?);
+            user_ids.insert(user_id?);
         }
 
         Ok(SyncEphemeralRoomEvent {
-            content: ruma::events::typing::TypingEventContent { user_ids },
+            content: ruma::events::typing::TypingEventContent {
+                user_ids: user_ids.into_iter().collect(),
+            },
         })
     }
 
@@ -320,19 +338,19 @@ impl RoomEdus {
 
         let count = globals.next_count()?.to_be_bytes();
 
-        let mut presence_id = room_id.to_string().as_bytes().to_vec();
+        let mut presence_id = room_id.as_bytes().to_vec();
         presence_id.push(0xff);
         presence_id.extend_from_slice(&count);
         presence_id.push(0xff);
-        presence_id.extend_from_slice(&presence.sender.to_string().as_bytes());
+        presence_id.extend_from_slice(&presence.sender.as_bytes());
 
         self.presenceid_presence.insert(
-            presence_id,
-            &*serde_json::to_string(&presence).expect("PresenceEvent can be serialized"),
+            &presence_id,
+            &serde_json::to_vec(&presence).expect("PresenceEvent can be serialized"),
         )?;
 
         self.userid_lastpresenceupdate.insert(
-            &user_id.to_string().as_bytes(),
+            user_id.as_bytes(),
             &utils::millis_since_unix_epoch().to_be_bytes(),
         )?;
 
@@ -340,9 +358,10 @@ impl RoomEdus {
     }
 
     /// Resets the presence timeout, so the user will stay in their current presence state.
+    #[tracing::instrument(skip(self))]
     pub fn ping_presence(&self, user_id: &UserId) -> Result<()> {
         self.userid_lastpresenceupdate.insert(
-            &user_id.to_string().as_bytes(),
+            &user_id.as_bytes(),
             &utils::millis_since_unix_epoch().to_be_bytes(),
         )?;
 
@@ -352,7 +371,7 @@ impl RoomEdus {
     /// Returns the timestamp of the last presence update of this user in millis since the unix epoch.
     pub fn last_presence_update(&self, user_id: &UserId) -> Result<Option<u64>> {
         self.userid_lastpresenceupdate
-            .get(&user_id.to_string().as_bytes())?
+            .get(&user_id.as_bytes())?
             .map(|bytes| {
                 utils::u64_from_bytes(&bytes).map_err(|_| {
                     Error::bad_database("Invalid timestamp in userid_lastpresenceupdate.")
@@ -361,8 +380,49 @@ impl RoomEdus {
             .transpose()
     }
 
+    pub fn get_last_presence_event(
+        &self,
+        user_id: &UserId,
+        room_id: &RoomId,
+    ) -> Result<Option<PresenceEvent>> {
+        let last_update = match self.last_presence_update(user_id)? {
+            Some(last) => last,
+            None => return Ok(None),
+        };
+
+        let mut presence_id = room_id.as_bytes().to_vec();
+        presence_id.push(0xff);
+        presence_id.extend_from_slice(&last_update.to_be_bytes());
+        presence_id.push(0xff);
+        presence_id.extend_from_slice(&user_id.as_bytes());
+
+        self.presenceid_presence
+            .get(&presence_id)?
+            .map(|value| {
+                let mut presence = serde_json::from_slice::<PresenceEvent>(&value)
+                    .map_err(|_| Error::bad_database("Invalid presence event in db."))?;
+                let current_timestamp: UInt = utils::millis_since_unix_epoch()
+                    .try_into()
+                    .expect("time is valid");
+
+                if presence.content.presence == PresenceState::Online {
+                    // Don't set last_active_ago when the user is online
+                    presence.content.last_active_ago = None;
+                } else {
+                    // Convert from timestamp to duration
+                    presence.content.last_active_ago = presence
+                        .content
+                        .last_active_ago
+                        .map(|timestamp| current_timestamp - timestamp);
+                }
+
+                Ok(presence)
+            })
+            .transpose()
+    }
+
     /// Sets all users to offline who have been quiet for too long.
-    pub fn presence_maintain(
+    fn presence_maintain(
         &self,
         rooms: &super::Rooms,
         globals: &super::super::globals::Globals,
@@ -372,7 +432,6 @@ impl RoomEdus {
         for (user_id_bytes, last_timestamp) in self
             .userid_lastpresenceupdate
             .iter()
-            .filter_map(|r| r.ok())
             .filter_map(|(k, bytes)| {
                 Some((
                     k,
@@ -383,11 +442,9 @@ impl RoomEdus {
                         .ok()?,
                 ))
             })
-            .take_while(|(_, timestamp)| current_timestamp - timestamp > 5 * 60_000)
+            .take_while(|(_, timestamp)| current_timestamp.saturating_sub(*timestamp) > 5 * 60_000)
         // 5 Minutes
         {
-            self.userid_lastpresenceupdate.remove(&user_id_bytes)?;
-
             // Send new presence events to set the user offline
             let count = globals.next_count()?.to_be_bytes();
             let user_id = utils::string_from_bytes(&user_id_bytes)
@@ -397,15 +454,15 @@ impl RoomEdus {
                 .try_into()
                 .map_err(|_| Error::bad_database("Invalid UserId in userid_lastpresenceupdate."))?;
             for room_id in rooms.rooms_joined(&user_id).filter_map(|r| r.ok()) {
-                let mut presence_id = room_id.to_string().as_bytes().to_vec();
+                let mut presence_id = room_id.as_bytes().to_vec();
                 presence_id.push(0xff);
                 presence_id.extend_from_slice(&count);
                 presence_id.push(0xff);
                 presence_id.extend_from_slice(&user_id_bytes);
 
                 self.presenceid_presence.insert(
-                    presence_id,
-                    &*serde_json::to_string(&PresenceEvent {
+                    &presence_id,
+                    &serde_json::to_vec(&PresenceEvent {
                         content: PresenceEventContent {
                             avatar_url: None,
                             currently_active: None,
@@ -421,12 +478,18 @@ impl RoomEdus {
                     .expect("PresenceEvent can be serialized"),
                 )?;
             }
+
+            self.userid_lastpresenceupdate.insert(
+                &user_id.as_bytes(),
+                &utils::millis_since_unix_epoch().to_be_bytes(),
+            )?;
         }
 
         Ok(())
     }
 
     /// Returns an iterator over the most recent presence updates that happened after the event with id `since`.
+    #[tracing::instrument(skip(self, globals, rooms))]
     pub fn presence_since(
         &self,
         room_id: &RoomId,
@@ -434,9 +497,9 @@ impl RoomEdus {
         rooms: &super::Rooms,
         globals: &super::super::globals::Globals,
     ) -> Result<HashMap<UserId, PresenceEvent>> {
-        self.presence_maintain(rooms, globals)?;
+        //self.presence_maintain(rooms, globals)?;
 
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let mut first_possible_edu = prefix.clone();
@@ -445,8 +508,7 @@ impl RoomEdus {
 
         for (key, value) in self
             .presenceid_presence
-            .range(&*first_possible_edu..)
-            .filter_map(|r| r.ok())
+            .iter_from(&*first_possible_edu, false)
             .take_while(|(key, _)| key.starts_with(&prefix))
         {
             let user_id = UserId::try_from(

src/database/sending.rs

@@ -1,160 +1,221 @@
-use std::{collections::HashMap, convert::TryFrom, time::SystemTime};
+use std::{
+    collections::{BTreeMap, HashMap},
+    convert::{TryFrom, TryInto},
+    fmt::Debug,
+    sync::Arc,
+    time::{Duration, Instant},
+};
 
-use crate::{server_server, utils, Error, PduEvent, Result};
+use crate::{
+    appservice_server, database::pusher, server_server, utils, Database, Error, PduEvent, Result,
+};
 use federation::transactions::send_transaction_message;
-use log::{debug, warn};
-use rocket::futures::stream::{FuturesUnordered, StreamExt};
-use ruma::{api::federation, ServerName};
-use sled::IVec;
-use tokio::select;
+use ring::digest;
+use rocket::futures::{
+    channel::mpsc,
+    stream::{FuturesUnordered, StreamExt},
+};
+use ruma::{
+    api::{
+        appservice,
+        federation::{
+            self,
+            transactions::edu::{Edu, ReceiptContent, ReceiptData, ReceiptMap},
+        },
+        OutgoingRequest,
+    },
+    events::{push_rules, AnySyncEphemeralRoomEvent, EventType},
+    push,
+    receipt::ReceiptType,
+    MilliSecondsSinceUnixEpoch, ServerName, UInt, UserId,
+};
+use tokio::{
+    select,
+    sync::{RwLock, Semaphore},
+};
+use tracing::{error, warn};
+
+use super::abstraction::Tree;
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum OutgoingKind {
+    Appservice(Box<ServerName>),
+    Push(Vec<u8>, Vec<u8>), // user and pushkey
+    Normal(Box<ServerName>),
+}
+
+impl OutgoingKind {
+    #[tracing::instrument(skip(self))]
+    pub fn get_prefix(&self) -> Vec<u8> {
+        let mut prefix = match self {
+            OutgoingKind::Appservice(server) => {
+                let mut p = b"+".to_vec();
+                p.extend_from_slice(server.as_bytes());
+                p
+            }
+            OutgoingKind::Push(user, pushkey) => {
+                let mut p = b"$".to_vec();
+                p.extend_from_slice(&user);
+                p.push(0xff);
+                p.extend_from_slice(&pushkey);
+                p
+            }
+            OutgoingKind::Normal(server) => {
+                let mut p = Vec::new();
+                p.extend_from_slice(server.as_bytes());
+                p
+            }
+        };
+        prefix.push(0xff);
+
+        prefix
+    }
+}
+
+#[derive(Clone, Debug, PartialEq, Eq, Hash)]
+pub enum SendingEventType {
+    Pdu(Vec<u8>),
+    Edu(Vec<u8>),
+}
 
-#[derive(Clone)]
 pub struct Sending {
     /// The state for a given state hash.
-    pub(super) servernamepduids: sled::Tree, // ServernamePduId = ServerName + PduId
-    pub(super) servercurrentpdus: sled::Tree, // ServerCurrentPdus = ServerName + PduId (pduid can be empty for reservation)
+    pub(super) servername_educount: Arc<dyn Tree>, // EduCount: Count of last EDU sync
+    pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / * (for edus), Data = EDU content
+    pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / * (for edus), Data = EDU content
+    pub(super) maximum_requests: Arc<Semaphore>,
+    pub sender: mpsc::UnboundedSender<(Vec<u8>, Vec<u8>)>,
+}
+
+enum TransactionStatus {
+    Running,
+    Failed(u32, Instant), // number of times failed, time of last failure
+    Retrying(u32),        // number of times failed
 }
 
 impl Sending {
-    pub fn start_handler(&self, globals: &super::globals::Globals, rooms: &super::rooms::Rooms) {
-        let servernamepduids = self.servernamepduids.clone();
-        let servercurrentpdus = self.servercurrentpdus.clone();
-        let rooms = rooms.clone();
-        let globals = globals.clone();
-
+    pub fn start_handler(
+        &self,
+        db: Arc<RwLock<Database>>,
+        mut receiver: mpsc::UnboundedReceiver<(Vec<u8>, Vec<u8>)>,
+    ) {
         tokio::spawn(async move {
             let mut futures = FuturesUnordered::new();
 
+            let mut current_transaction_status = HashMap::<Vec<u8>, TransactionStatus>::new();
+
             // Retry requests we could not finish yet
-            let mut current_transactions = HashMap::new();
+            let mut initial_transactions = HashMap::<OutgoingKind, Vec<SendingEventType>>::new();
 
-            for (server, pdu) in servercurrentpdus
+            let guard = db.read().await;
+
+            for (key, outgoing_kind, event) in guard
+                .sending
+                .servercurrentevent_data
                 .iter()
-                .filter_map(|r| r.ok())
-                .map(|(key, _)| {
-                    let mut parts = key.splitn(2, |&b| b == 0xff);
-                    let server = parts.next().expect("splitn always returns one element");
-                    let pdu = parts.next().ok_or_else(|| {
-                        Error::bad_database("Invalid bytes in servercurrentpdus.")
-                    })?;
-
-                    Ok::<_, Error>((
-                        Box::<ServerName>::try_from(utils::string_from_bytes(&server).map_err(
-                            |_| {
-                                Error::bad_database(
-                                    "Invalid server bytes in server_currenttransaction",
-                                )
-                            },
-                        )?)
-                        .map_err(|_| {
-                            Error::bad_database(
-                                "Invalid server string in server_currenttransaction",
-                            )
-                        })?,
-                        IVec::from(pdu),
-                    ))
+                .filter_map(|(key, v)| {
+                    Self::parse_servercurrentevent(&key, v)
+                        .ok()
+                        .map(|(k, e)| (key, k, e))
                 })
-                .filter_map(|r| r.ok())
-                .filter(|(_, pdu)| !pdu.is_empty()) // Skip reservation key
-                .take(50)
-            // This should not contain more than 50 anyway
             {
-                current_transactions
-                    .entry(server)
-                    .or_insert_with(Vec::new)
-                    .push(pdu);
+                let entry = initial_transactions
+                    .entry(outgoing_kind.clone())
+                    .or_insert_with(Vec::new);
+
+                if entry.len() > 30 {
+                    warn!(
+                        "Dropping some current events: {:?} {:?} {:?}",
+                        key, outgoing_kind, event
+                    );
+                    guard.sending.servercurrentevent_data.remove(&key).unwrap();
+                    continue;
+                }
+
+                entry.push(event);
             }
 
-            for (server, pdus) in current_transactions {
-                futures.push(Self::handle_event(server, pdus, &globals, &rooms));
+            drop(guard);
+
+            for (outgoing_kind, events) in initial_transactions {
+                current_transaction_status
+                    .insert(outgoing_kind.get_prefix(), TransactionStatus::Running);
+                futures.push(Self::handle_events(
+                    outgoing_kind.clone(),
+                    events,
+                    Arc::clone(&db),
+                ));
             }
 
-            let mut subscriber = servernamepduids.watch_prefix(b"");
             loop {
                 select! {
-                    Some(server) = futures.next() => {
-                        debug!("sending response: {:?}", &server);
-                        match server {
-                            Ok((server, _response)) => {
-                                let mut prefix = server.as_bytes().to_vec();
-                                prefix.push(0xff);
+                    Some(response) = futures.next() => {
+                        match response {
+                            Ok(outgoing_kind) => {
+                                let guard = db.read().await;
 
-                                for key in servercurrentpdus
-                                    .scan_prefix(&prefix)
-                                    .keys()
-                                    .filter_map(|r| r.ok())
+                                let prefix = outgoing_kind.get_prefix();
+                                for (key, _) in guard.sending.servercurrentevent_data
+                                    .scan_prefix(prefix.clone())
                                 {
-                                    // Don't remove reservation yet
-                                    if prefix.len() != key.len() {
-                                        servercurrentpdus.remove(key).unwrap();
-                                    }
+                                    guard.sending.servercurrentevent_data.remove(&key).unwrap();
                                 }
 
                                 // Find events that have been added since starting the last request
-                                let new_pdus = servernamepduids
-                                    .scan_prefix(&prefix)
-                                    .keys()
-                                    .filter_map(|r| r.ok())
-                                    .map(|k| {
-                                        k.subslice(prefix.len(), k.len() - prefix.len())
+                                let new_events = guard.sending.servernameevent_data
+                                    .scan_prefix(prefix.clone())
+                                    .filter_map(|(k, v)| {
+                                        Self::parse_servercurrentevent(&k, v).ok().map(|ev| (ev, k))
                                     })
-                                    .take(50)
+                                    .take(30)
                                     .collect::<Vec<_>>();
 
-                                if !new_pdus.is_empty() {
-                                    for pdu_id in &new_pdus {
-                                        let mut current_key = prefix.clone();
-                                        current_key.extend_from_slice(pdu_id);
-                                        servercurrentpdus.insert(&current_key, &[]).unwrap();
-                                        servernamepduids.remove(&current_key).unwrap();
+                                // TODO: find edus
+
+                                if !new_events.is_empty() {
+                                    // Insert pdus we found
+                                    for (e, key) in &new_events {
+                                        let value = if let SendingEventType::Edu(value) = &e.1 { &**value } else { &[] };
+                                        guard.sending.servercurrentevent_data.insert(&key, value).unwrap();
+                                        guard.sending.servernameevent_data.remove(&key).unwrap();
                                     }
 
-                                    futures.push(Self::handle_event(server, new_pdus, &globals, &rooms));
+                                    drop(guard);
+
+                                    futures.push(
+                                        Self::handle_events(
+                                            outgoing_kind.clone(),
+                                            new_events.into_iter().map(|(event, _)| event.1).collect(),
+                                            Arc::clone(&db),
+                                        )
+                                    );
                                 } else {
-                                    servercurrentpdus.remove(&prefix).unwrap();
-                                    // servercurrentpdus with the prefix should be empty now
+                                    current_transaction_status.remove(&prefix);
                                 }
                             }
-                            Err((server, e)) => {
-                                warn!("Couldn't send transaction to {}: {}", server, e)
-                                // TODO: exponential backoff
+                            Err((outgoing_kind, _)) => {
+                                current_transaction_status.entry(outgoing_kind.get_prefix()).and_modify(|e| *e = match e {
+                                    TransactionStatus::Running => TransactionStatus::Failed(1, Instant::now()),
+                                    TransactionStatus::Retrying(n) => TransactionStatus::Failed(*n+1, Instant::now()),
+                                    TransactionStatus::Failed(_, _) => {
+                                        error!("Request that was not even running failed?!");
+                                        return
+                                    },
+                                });
                             }
                         };
                     },
-                    Some(event) = &mut subscriber => {
-                        if let sled::Event::Insert { key, .. } = event {
-                            let servernamepduid = key.clone();
-                            let mut parts = servernamepduid.splitn(2, |&b| b == 0xff);
+                    Some((key, value)) = receiver.next() => {
+                        if let Ok((outgoing_kind, event)) = Self::parse_servercurrentevent(&key, value) {
+                            let guard = db.read().await;
 
-                            if let Some((server, pdu_id)) = utils::string_from_bytes(
-                                    parts
-                                        .next()
-                                        .expect("splitn will always return 1 or more elements"),
-                                )
-                                .map_err(|_| Error::bad_database("ServerName in servernamepduid bytes are invalid."))
-                                .and_then(|server_str| Box::<ServerName>::try_from(server_str)
-                                    .map_err(|_| Error::bad_database("ServerName in servernamepduid is invalid.")))
-                                .ok()
-                                .and_then(|server| parts
-                                    .next()
-                                    .ok_or_else(|| Error::bad_database("Invalid servernamepduid in db."))
-                                    .ok()
-                                    .map(|pdu_id| (server, pdu_id))
-                                )
-                                // TODO: exponential backoff
-                                .filter(|(server, _)| {
-                                    let mut prefix = server.to_string().as_bytes().to_vec();
-                                    prefix.push(0xff);
-
-                                    servercurrentpdus
-                                        .compare_and_swap(prefix, Option::<&[u8]>::None, Some(&[])) // Try to reserve
-                                        == Ok(Ok(()))
-                                })
-                            {
-                                servercurrentpdus.insert(&key, &[]).unwrap();
-                                servernamepduids.remove(&key).unwrap();
-
-                                futures.push(Self::handle_event(server, vec![pdu_id.into()], &globals, &rooms));
+                            if let Ok(Some(events)) = Self::select_events(
+                                &outgoing_kind,
+                                vec![(event, key)],
+                                &mut current_transaction_status,
+                                &guard
+                            ) {
+                                futures.push(Self::handle_events(outgoing_kind, events, Arc::clone(&db)));
                             }
                         }
                     }
@@ -163,65 +224,545 @@ impl Sending {
         });
     }
 
-    pub fn send_pdu(&self, server: &ServerName, pdu_id: &[u8]) -> Result<()> {
-        let mut key = server.as_bytes().to_vec();
+    #[tracing::instrument(skip(outgoing_kind, new_events, current_transaction_status, db))]
+    fn select_events(
+        outgoing_kind: &OutgoingKind,
+        new_events: Vec<(SendingEventType, Vec<u8>)>, // Events we want to send: event and full key
+        current_transaction_status: &mut HashMap<Vec<u8>, TransactionStatus>,
+        db: &Database,
+    ) -> Result<Option<Vec<SendingEventType>>> {
+        let mut retry = false;
+        let mut allow = true;
+
+        let prefix = outgoing_kind.get_prefix();
+        let entry = current_transaction_status.entry(prefix.clone());
+
+        entry
+            .and_modify(|e| match e {
+                TransactionStatus::Running | TransactionStatus::Retrying(_) => {
+                    allow = false; // already running
+                }
+                TransactionStatus::Failed(tries, time) => {
+                    // Fail if a request has failed recently (exponential backoff)
+                    let mut min_elapsed_duration = Duration::from_secs(30) * (*tries) * (*tries);
+                    if min_elapsed_duration > Duration::from_secs(60 * 60 * 24) {
+                        min_elapsed_duration = Duration::from_secs(60 * 60 * 24);
+                    }
+
+                    if time.elapsed() < min_elapsed_duration {
+                        allow = false;
+                    } else {
+                        retry = true;
+                        *e = TransactionStatus::Retrying(*tries);
+                    }
+                }
+            })
+            .or_insert(TransactionStatus::Running);
+
+        if !allow {
+            return Ok(None);
+        }
+
+        let mut events = Vec::new();
+
+        if retry {
+            // We retry the previous transaction
+            for (key, value) in db.sending.servercurrentevent_data.scan_prefix(prefix) {
+                if let Ok((_, e)) = Self::parse_servercurrentevent(&key, value) {
+                    events.push(e);
+                }
+            }
+        } else {
+            for (e, full_key) in new_events {
+                let value = if let SendingEventType::Edu(value) = &e {
+                    &**value
+                } else {
+                    &[][..]
+                };
+                db.sending
+                    .servercurrentevent_data
+                    .insert(&full_key, value)?;
+
+                // If it was a PDU we have to unqueue it
+                // TODO: don't try to unqueue EDUs
+                db.sending.servernameevent_data.remove(&full_key)?;
+
+                events.push(e);
+            }
+
+            if let OutgoingKind::Normal(server_name) = outgoing_kind {
+                if let Ok((select_edus, last_count)) = Self::select_edus(db, server_name) {
+                    events.extend(select_edus.into_iter().map(SendingEventType::Edu));
+
+                    db.sending
+                        .servername_educount
+                        .insert(server_name.as_bytes(), &last_count.to_be_bytes())?;
+                }
+            }
+        }
+
+        Ok(Some(events))
+    }
+
+    #[tracing::instrument(skip(db, server))]
+    pub fn select_edus(db: &Database, server: &ServerName) -> Result<(Vec<Vec<u8>>, u64)> {
+        // u64: count of last edu
+        let since = db
+            .sending
+            .servername_educount
+            .get(server.as_bytes())?
+            .map_or(Ok(0), |bytes| {
+                utils::u64_from_bytes(&bytes)
+                    .map_err(|_| Error::bad_database("Invalid u64 in servername_educount."))
+            })?;
+        let mut events = Vec::new();
+        let mut max_edu_count = since;
+        'outer: for room_id in db.rooms.server_rooms(server) {
+            let room_id = room_id?;
+            for r in db.rooms.edus.readreceipts_since(&room_id, since) {
+                let (user_id, count, read_receipt) = r?;
+
+                if count > max_edu_count {
+                    max_edu_count = count;
+                }
+
+                if user_id.server_name() != db.globals.server_name() {
+                    continue;
+                }
+
+                let event =
+                    serde_json::from_str::<AnySyncEphemeralRoomEvent>(&read_receipt.json().get())
+                        .map_err(|_| Error::bad_database("Invalid edu event in read_receipts."))?;
+                let federation_event = match event {
+                    AnySyncEphemeralRoomEvent::Receipt(r) => {
+                        let mut read = BTreeMap::new();
+
+                        let (event_id, mut receipt) = r
+                            .content
+                            .0
+                            .into_iter()
+                            .next()
+                            .expect("we only use one event per read receipt");
+                        let receipt = receipt
+                            .remove(&ReceiptType::Read)
+                            .expect("our read receipts always set this")
+                            .remove(&user_id)
+                            .expect("our read receipts always have the user here");
+
+                        read.insert(
+                            user_id,
+                            ReceiptData {
+                                data: receipt.clone(),
+                                event_ids: vec![event_id.clone()],
+                            },
+                        );
+
+                        let receipt_map = ReceiptMap { read };
+
+                        let mut receipts = BTreeMap::new();
+                        receipts.insert(room_id.clone(), receipt_map);
+
+                        Edu::Receipt(ReceiptContent { receipts })
+                    }
+                    _ => {
+                        Error::bad_database("Invalid event type in read_receipts");
+                        continue;
+                    }
+                };
+
+                events.push(serde_json::to_vec(&federation_event).expect("json can be serialized"));
+
+                if events.len() >= 20 {
+                    break 'outer;
+                }
+            }
+        }
+
+        Ok((events, max_edu_count))
+    }
+
+    #[tracing::instrument(skip(self, pdu_id, senderkey))]
+    pub fn send_push_pdu(&self, pdu_id: &[u8], senderkey: Vec<u8>) -> Result<()> {
+        let mut key = b"$".to_vec();
+        key.extend_from_slice(&senderkey);
         key.push(0xff);
         key.extend_from_slice(pdu_id);
-        self.servernamepduids.insert(key, b"")?;
+        self.servernameevent_data.insert(&key, &[])?;
+        self.sender.unbounded_send((key, vec![])).unwrap();
 
         Ok(())
     }
 
-    async fn handle_event(
-        server: Box<ServerName>,
-        pdu_ids: Vec<IVec>,
-        globals: &super::globals::Globals,
-        rooms: &super::rooms::Rooms,
-    ) -> std::result::Result<
-        (Box<ServerName>, send_transaction_message::v1::Response),
-        (Box<ServerName>, Error),
-    > {
-        let pdu_jsons = pdu_ids
-            .iter()
-            .map(|pdu_id| {
-                Ok::<_, (Box<ServerName>, Error)>(
-                    // TODO: check room version and remove event_id if needed
-                    serde_json::from_str(
-                        PduEvent::convert_to_outgoing_federation_event(
-                            rooms
-                                .get_pdu_json_from_id(pdu_id)
-                                .map_err(|e| (server.clone(), e))?
+    #[tracing::instrument(skip(self, server, pdu_id))]
+    pub fn send_pdu(&self, server: &ServerName, pdu_id: &[u8]) -> Result<()> {
+        let mut key = server.as_bytes().to_vec();
+        key.push(0xff);
+        key.extend_from_slice(pdu_id);
+        self.servernameevent_data.insert(&key, &[])?;
+        self.sender.unbounded_send((key, vec![])).unwrap();
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, server, serialized))]
+    pub fn send_reliable_edu(&self, server: &ServerName, serialized: Vec<u8>) -> Result<()> {
+        let mut key = server.as_bytes().to_vec();
+        key.push(0xff);
+        key.push(b'*');
+        self.servernameevent_data.insert(&key, &serialized)?;
+        self.sender.unbounded_send((key, serialized)).unwrap();
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self))]
+    pub fn send_pdu_appservice(&self, appservice_id: &str, pdu_id: &[u8]) -> Result<()> {
+        let mut key = b"+".to_vec();
+        key.extend_from_slice(appservice_id.as_bytes());
+        key.push(0xff);
+        key.extend_from_slice(pdu_id);
+        self.servernameevent_data.insert(&key, &[])?;
+        self.sender.unbounded_send((key, vec![])).unwrap();
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(keys))]
+    fn calculate_hash(keys: &[&[u8]]) -> Vec<u8> {
+        // We only hash the pdu's event ids, not the whole pdu
+        let bytes = keys.join(&0xff);
+        let hash = digest::digest(&digest::SHA256, &bytes);
+        hash.as_ref().to_owned()
+    }
+
+    #[tracing::instrument(skip(db, events, kind))]
+    async fn handle_events(
+        kind: OutgoingKind,
+        events: Vec<SendingEventType>,
+        db: Arc<RwLock<Database>>,
+    ) -> std::result::Result<OutgoingKind, (OutgoingKind, Error)> {
+        let db = db.read().await;
+
+        match &kind {
+            OutgoingKind::Appservice(server) => {
+                let mut pdu_jsons = Vec::new();
+
+                for event in &events {
+                    match event {
+                        SendingEventType::Pdu(pdu_id) => {
+                            pdu_jsons.push(db.rooms
+                                .get_pdu_from_id(&pdu_id)
+                                .map_err(|e| (kind.clone(), e))?
                                 .ok_or_else(|| {
                                     (
-                                        server.clone(),
+                                        kind.clone(),
                                         Error::bad_database(
-                                            "Event in servernamepduids not found in db.",
+                                            "[Appservice] Event in servernameevent_data not found in db.",
                                         ),
                                     )
-                                })?,
-                        )
-                        .json()
-                        .get(),
-                    )
-                    .expect("Raw<..> is always valid"),
-                )
-            })
-            .filter_map(|r| r.ok())
-            .collect::<Vec<_>>();
+                                })?
+                                .to_room_event())
+                        }
+                        SendingEventType::Edu(_) => {
+                            // Appservices don't need EDUs (?)
+                        }
+                    }
+                }
 
-        server_server::send_request(
-            &globals,
-            server.clone(),
-            send_transaction_message::v1::Request {
-                origin: globals.server_name(),
-                pdus: &pdu_jsons,
-                edus: &[],
-                origin_server_ts: SystemTime::now(),
-                transaction_id: &utils::random_string(16),
-            },
-        )
-        .await
-        .map(|response| (server.clone(), response))
-        .map_err(|e| (server, e))
+                let permit = db.sending.maximum_requests.acquire().await;
+
+                let response = appservice_server::send_request(
+                    &db.globals,
+                    db.appservice
+                        .get_registration(server.as_str())
+                        .unwrap()
+                        .unwrap(), // TODO: handle error
+                    appservice::event::push_events::v1::Request {
+                        events: &pdu_jsons,
+                        txn_id: &base64::encode_config(
+                            Self::calculate_hash(
+                                &events
+                                    .iter()
+                                    .map(|e| match e {
+                                        SendingEventType::Edu(b) | SendingEventType::Pdu(b) => &**b,
+                                    })
+                                    .collect::<Vec<_>>(),
+                            ),
+                            base64::URL_SAFE_NO_PAD,
+                        ),
+                    },
+                )
+                .await
+                .map(|_response| kind.clone())
+                .map_err(|e| (kind, e));
+
+                drop(permit);
+
+                response
+            }
+            OutgoingKind::Push(user, pushkey) => {
+                let mut pdus = Vec::new();
+
+                for event in &events {
+                    match event {
+                        SendingEventType::Pdu(pdu_id) => {
+                            pdus.push(
+                                db.rooms
+                                    .get_pdu_from_id(&pdu_id)
+                                    .map_err(|e| (kind.clone(), e))?
+                                    .ok_or_else(|| {
+                                        (
+                                            kind.clone(),
+                                            Error::bad_database(
+                                                "[Push] Event in servernamevent_datas not found in db.",
+                                            ),
+                                        )
+                                    })?,
+                            );
+                        }
+                        SendingEventType::Edu(_) => {
+                            // Push gateways don't need EDUs (?)
+                        }
+                    }
+                }
+
+                for pdu in pdus {
+                    // Redacted events are not notification targets (we don't send push for them)
+                    if pdu.unsigned.get("redacted_because").is_some() {
+                        continue;
+                    }
+
+                    let userid =
+                        UserId::try_from(utils::string_from_bytes(user).map_err(|_| {
+                            (
+                                kind.clone(),
+                                Error::bad_database("Invalid push user string in db."),
+                            )
+                        })?)
+                        .map_err(|_| {
+                            (
+                                kind.clone(),
+                                Error::bad_database("Invalid push user id in db."),
+                            )
+                        })?;
+
+                    let mut senderkey = user.clone();
+                    senderkey.push(0xff);
+                    senderkey.extend_from_slice(pushkey);
+
+                    let pusher = match db
+                        .pusher
+                        .get_pusher(&senderkey)
+                        .map_err(|e| (OutgoingKind::Push(user.clone(), pushkey.clone()), e))?
+                    {
+                        Some(pusher) => pusher,
+                        None => continue,
+                    };
+
+                    let rules_for_user = db
+                        .account_data
+                        .get::<push_rules::PushRulesEvent>(None, &userid, EventType::PushRules)
+                        .unwrap_or_default()
+                        .map(|ev| ev.content.global)
+                        .unwrap_or_else(|| push::Ruleset::server_default(&userid));
+
+                    let unread: UInt = db
+                        .rooms
+                        .notification_count(&userid, &pdu.room_id)
+                        .map_err(|e| (kind.clone(), e))?
+                        .try_into()
+                        .expect("notifiation count can't go that high");
+
+                    let permit = db.sending.maximum_requests.acquire().await;
+
+                    let _response = pusher::send_push_notice(
+                        &userid,
+                        unread,
+                        &pusher,
+                        rules_for_user,
+                        &pdu,
+                        &db,
+                    )
+                    .await
+                    .map(|_response| kind.clone())
+                    .map_err(|e| (kind.clone(), e));
+
+                    drop(permit);
+                }
+                Ok(OutgoingKind::Push(user.clone(), pushkey.clone()))
+            }
+            OutgoingKind::Normal(server) => {
+                let mut edu_jsons = Vec::new();
+                let mut pdu_jsons = Vec::new();
+
+                for event in &events {
+                    match event {
+                        SendingEventType::Pdu(pdu_id) => {
+                            // TODO: check room version and remove event_id if needed
+                            let raw = PduEvent::convert_to_outgoing_federation_event(
+                                db.rooms
+                                    .get_pdu_json_from_id(&pdu_id)
+                                    .map_err(|e| (OutgoingKind::Normal(server.clone()), e))?
+                                    .ok_or_else(|| {
+                                        (
+                                            OutgoingKind::Normal(server.clone()),
+                                            Error::bad_database(
+                                                "[Normal] Event in servernamevent_datas not found in db.",
+                                            ),
+                                        )
+                                    })?,
+                            );
+                            pdu_jsons.push(raw);
+                        }
+                        SendingEventType::Edu(edu) => {
+                            if let Ok(raw) = serde_json::from_slice(edu) {
+                                edu_jsons.push(raw);
+                            }
+                        }
+                    }
+                }
+
+                let permit = db.sending.maximum_requests.acquire().await;
+
+                let response = server_server::send_request(
+                    &db.globals,
+                    &*server,
+                    send_transaction_message::v1::Request {
+                        origin: db.globals.server_name(),
+                        pdus: &pdu_jsons,
+                        edus: &edu_jsons,
+                        origin_server_ts: MilliSecondsSinceUnixEpoch::now(),
+                        transaction_id: &base64::encode_config(
+                            Self::calculate_hash(
+                                &events
+                                    .iter()
+                                    .map(|e| match e {
+                                        SendingEventType::Edu(b) | SendingEventType::Pdu(b) => &**b,
+                                    })
+                                    .collect::<Vec<_>>(),
+                            ),
+                            base64::URL_SAFE_NO_PAD,
+                        ),
+                    },
+                )
+                .await
+                .map(|response| {
+                    for pdu in response.pdus {
+                        if pdu.1.is_err() {
+                            warn!("Failed to send to {}: {:?}", server, pdu);
+                        }
+                    }
+                    kind.clone()
+                })
+                .map_err(|e| (kind, e));
+
+                drop(permit);
+
+                response
+            }
+        }
+    }
+
+    #[tracing::instrument(skip(key))]
+    fn parse_servercurrentevent(
+        key: &[u8],
+        value: Vec<u8>,
+    ) -> Result<(OutgoingKind, SendingEventType)> {
+        // Appservices start with a plus
+        Ok::<_, Error>(if key.starts_with(b"+") {
+            let mut parts = key[1..].splitn(2, |&b| b == 0xff);
+
+            let server = parts.next().expect("splitn always returns one element");
+            let event = parts
+                .next()
+                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+            let server = utils::string_from_bytes(&server).map_err(|_| {
+                Error::bad_database("Invalid server bytes in server_currenttransaction")
+            })?;
+
+            (
+                OutgoingKind::Appservice(Box::<ServerName>::try_from(server).map_err(|_| {
+                    Error::bad_database("Invalid server string in server_currenttransaction")
+                })?),
+                if event.starts_with(b"*") {
+                    SendingEventType::Edu(value)
+                } else {
+                    SendingEventType::Pdu(event.to_vec())
+                },
+            )
+        } else if key.starts_with(b"$") {
+            let mut parts = key[1..].splitn(3, |&b| b == 0xff);
+
+            let user = parts.next().expect("splitn always returns one element");
+            let pushkey = parts
+                .next()
+                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+            let event = parts
+                .next()
+                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+            (
+                OutgoingKind::Push(user.to_vec(), pushkey.to_vec()),
+                if event.starts_with(b"*") {
+                    SendingEventType::Edu(value)
+                } else {
+                    SendingEventType::Pdu(event.to_vec())
+                },
+            )
+        } else {
+            let mut parts = key.splitn(2, |&b| b == 0xff);
+
+            let server = parts.next().expect("splitn always returns one element");
+            let event = parts
+                .next()
+                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
+            let server = utils::string_from_bytes(&server).map_err(|_| {
+                Error::bad_database("Invalid server bytes in server_currenttransaction")
+            })?;
+
+            (
+                OutgoingKind::Normal(Box::<ServerName>::try_from(server).map_err(|_| {
+                    Error::bad_database("Invalid server string in server_currenttransaction")
+                })?),
+                if event.starts_with(b"*") {
+                    SendingEventType::Edu(event[1..].to_vec())
+                } else {
+                    SendingEventType::Pdu(event.to_vec())
+                },
+            )
+        })
+    }
+
+    #[tracing::instrument(skip(self, globals, destination, request))]
+    pub async fn send_federation_request<T: OutgoingRequest>(
+        &self,
+        globals: &crate::database::globals::Globals,
+        destination: &ServerName,
+        request: T,
+    ) -> Result<T::IncomingResponse>
+    where
+        T: Debug,
+    {
+        let permit = self.maximum_requests.acquire().await;
+        let response = server_server::send_request(globals, destination, request).await;
+        drop(permit);
+
+        response
+    }
+
+    #[tracing::instrument(skip(self, globals, registration, request))]
+    pub async fn send_appservice_request<T: OutgoingRequest>(
+        &self,
+        globals: &crate::database::globals::Globals,
+        registration: serde_yaml::Value,
+        request: T,
+    ) -> Result<T::IncomingResponse>
+    where
+        T: Debug,
+    {
+        let permit = self.maximum_requests.acquire().await;
+        let response = appservice_server::send_request(globals, registration, request).await;
+        drop(permit);
+
+        response
     }
 }

src/database/transaction_ids.rs

@@ -1,27 +1,29 @@
+use std::sync::Arc;
+
 use crate::Result;
 use ruma::{DeviceId, UserId};
-use sled::IVec;
 
-#[derive(Clone)]
+use super::abstraction::Tree;
+
 pub struct TransactionIds {
-    pub(super) userdevicetxnid_response: sled::Tree, // Response can be empty (/sendToDevice) or the event id (/send)
+    pub(super) userdevicetxnid_response: Arc<dyn Tree>, // Response can be empty (/sendToDevice) or the event id (/send)
 }
 
 impl TransactionIds {
     pub fn add_txnid(
         &self,
         user_id: &UserId,
-        device_id: &DeviceId,
+        device_id: Option<&DeviceId>,
         txn_id: &str,
         data: &[u8],
     ) -> Result<()> {
         let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(device_id.as_bytes());
+        key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default());
         key.push(0xff);
         key.extend_from_slice(txn_id.as_bytes());
 
-        self.userdevicetxnid_response.insert(key, data)?;
+        self.userdevicetxnid_response.insert(&key, data)?;
 
         Ok(())
     }
@@ -29,16 +31,16 @@ impl TransactionIds {
     pub fn existing_txnid(
         &self,
         user_id: &UserId,
-        device_id: &DeviceId,
+        device_id: Option<&DeviceId>,
         txn_id: &str,
-    ) -> Result<Option<IVec>> {
+    ) -> Result<Option<Vec<u8>>> {
         let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(device_id.as_bytes());
+        key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default());
         key.push(0xff);
         key.extend_from_slice(txn_id.as_bytes());
 
         // If there's no entry, this is a new transaction
-        Ok(self.userdevicetxnid_response.get(key)?)
+        self.userdevicetxnid_response.get(&key)
     }
 }

src/database/uiaa.rs

@@ -1,15 +1,20 @@
-use crate::{Error, Result};
+use std::sync::Arc;
+
+use crate::{client_server::SESSION_ID_LENGTH, utils, Error, Result};
 use ruma::{
     api::client::{
         error::ErrorKind,
         r0::uiaa::{IncomingAuthData, UiaaInfo},
     },
+    signatures::CanonicalJsonValue,
     DeviceId, UserId,
 };
 
-#[derive(Clone)]
+use super::abstraction::Tree;
+
 pub struct Uiaa {
-    pub(super) userdeviceid_uiaainfo: sled::Tree, // User-interactive authentication
+    pub(super) userdevicesessionid_uiaainfo: Arc<dyn Tree>, // User-interactive authentication
+    pub(super) userdevicesessionid_uiaarequest: Arc<dyn Tree>, // UiaaRequest = canonical json value
 }
 
 impl Uiaa {
@@ -19,8 +24,20 @@ impl Uiaa {
         user_id: &UserId,
         device_id: &DeviceId,
         uiaainfo: &UiaaInfo,
+        json_body: &CanonicalJsonValue,
     ) -> Result<()> {
-        self.update_uiaa_session(user_id, device_id, Some(uiaainfo))
+        self.set_uiaa_request(
+            user_id,
+            device_id,
+            uiaainfo.session.as_ref().expect("session should be set"), // TODO: better session error handling (why is it optional in ruma?)
+            json_body,
+        )?;
+        self.update_uiaa_session(
+            user_id,
+            device_id,
+            uiaainfo.session.as_ref().expect("session should be set"),
+            Some(uiaainfo),
+        )
     }
 
     pub fn try_auth(
@@ -40,11 +57,13 @@ impl Uiaa {
         {
             let mut uiaainfo = session
                 .as_ref()
-                .map(|session| {
-                    Ok::<_, Error>(self.get_uiaa_session(&user_id, &device_id, session)?)
-                })
+                .map(|session| self.get_uiaa_session(&user_id, &device_id, session))
                 .unwrap_or_else(|| Ok(uiaainfo.clone()))?;
 
+            if uiaainfo.session.is_none() {
+                uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
+            }
+
             // Find out what the user completed
             match &**kind {
                 "m.login.password" => {
@@ -130,35 +149,96 @@ impl Uiaa {
             }
 
             if !completed {
-                self.update_uiaa_session(user_id, device_id, Some(&uiaainfo))?;
+                self.update_uiaa_session(
+                    user_id,
+                    device_id,
+                    uiaainfo.session.as_ref().expect("session is always set"),
+                    Some(&uiaainfo),
+                )?;
                 return Ok((false, uiaainfo));
             }
 
             // UIAA was successful! Remove this session and return true
-            self.update_uiaa_session(user_id, device_id, None)?;
+            self.update_uiaa_session(
+                user_id,
+                device_id,
+                uiaainfo.session.as_ref().expect("session is always set"),
+                None,
+            )?;
             Ok((true, uiaainfo))
         } else {
             panic!("FallbackAcknowledgement is not supported yet");
         }
     }
 
+    fn set_uiaa_request(
+        &self,
+        user_id: &UserId,
+        device_id: &DeviceId,
+        session: &str,
+        request: &CanonicalJsonValue,
+    ) -> Result<()> {
+        let mut userdevicesessionid = user_id.as_bytes().to_vec();
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(device_id.as_bytes());
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(session.as_bytes());
+
+        self.userdevicesessionid_uiaarequest.insert(
+            &userdevicesessionid,
+            &serde_json::to_vec(request).expect("json value to vec always works"),
+        )?;
+
+        Ok(())
+    }
+
+    pub fn get_uiaa_request(
+        &self,
+        user_id: &UserId,
+        device_id: &DeviceId,
+        session: &str,
+    ) -> Result<Option<CanonicalJsonValue>> {
+        let mut userdevicesessionid = user_id.as_bytes().to_vec();
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(device_id.as_bytes());
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(session.as_bytes());
+
+        self.userdevicesessionid_uiaarequest
+            .get(&userdevicesessionid)?
+            .map_or(Ok(None), |bytes| {
+                Ok::<_, Error>(Some(
+                    serde_json::from_str::<CanonicalJsonValue>(
+                        &utils::string_from_bytes(&bytes).map_err(|_| {
+                            Error::bad_database("Invalid uiaa request bytes in db.")
+                        })?,
+                    )
+                    .map_err(|_| Error::bad_database("Invalid uiaa request in db."))?,
+                ))
+            })
+    }
+
     fn update_uiaa_session(
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
+        session: &str,
         uiaainfo: Option<&UiaaInfo>,
     ) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
-        userdeviceid.push(0xff);
-        userdeviceid.extend_from_slice(device_id.as_bytes());
+        let mut userdevicesessionid = user_id.as_bytes().to_vec();
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(device_id.as_bytes());
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(session.as_bytes());
 
         if let Some(uiaainfo) = uiaainfo {
-            self.userdeviceid_uiaainfo.insert(
-                &userdeviceid,
-                &*serde_json::to_string(&uiaainfo).expect("UiaaInfo::to_string always works"),
+            self.userdevicesessionid_uiaainfo.insert(
+                &userdevicesessionid,
+                &serde_json::to_vec(&uiaainfo).expect("UiaaInfo::to_vec always works"),
             )?;
         } else {
-            self.userdeviceid_uiaainfo.remove(&userdeviceid)?;
+            self.userdevicesessionid_uiaainfo
+                .remove(&userdevicesessionid)?;
         }
 
         Ok(())
@@ -170,14 +250,16 @@ impl Uiaa {
         device_id: &DeviceId,
         session: &str,
     ) -> Result<UiaaInfo> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
-        userdeviceid.push(0xff);
-        userdeviceid.extend_from_slice(device_id.as_bytes());
+        let mut userdevicesessionid = user_id.as_bytes().to_vec();
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(device_id.as_bytes());
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(session.as_bytes());
 
         let uiaainfo = serde_json::from_slice::<UiaaInfo>(
             &self
-                .userdeviceid_uiaainfo
-                .get(&userdeviceid)?
+                .userdevicesessionid_uiaainfo
+                .get(&userdevicesessionid)?
                 .ok_or(Error::BadRequest(
                     ErrorKind::Forbidden,
                     "UIAA session does not exist.",
@@ -185,18 +267,6 @@ impl Uiaa {
         )
         .map_err(|_| Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid."))?;
 
-        if uiaainfo
-            .session
-            .as_ref()
-            .filter(|&s| s == session)
-            .is_none()
-        {
-            return Err(Error::BadRequest(
-                ErrorKind::Forbidden,
-                "UIAA session token invalid.",
-            ));
-        }
-
         Ok(uiaainfo)
     }
 }

src/database/users.rs

@@ -1,51 +1,51 @@
 use crate::{utils, Error, Result};
-use js_int::UInt;
 use ruma::{
-    api::client::{
-        error::ErrorKind,
-        r0::{
-            device::Device,
-            keys::{CrossSigningKey, OneTimeKey},
-        },
-    },
-    encryption::DeviceKeys,
+    api::client::{error::ErrorKind, r0::device::Device},
+    encryption::{CrossSigningKey, DeviceKeys, OneTimeKey},
     events::{AnyToDeviceEvent, EventType},
+    identifiers::MxcUri,
     serde::Raw,
-    DeviceId, DeviceKeyAlgorithm, DeviceKeyId, UserId,
+    DeviceId, DeviceKeyAlgorithm, DeviceKeyId, MilliSecondsSinceUnixEpoch, UInt, UserId,
 };
-use std::{collections::BTreeMap, convert::TryFrom, mem, time::SystemTime};
+use std::{collections::BTreeMap, convert::TryFrom, mem, sync::Arc};
+use tracing::warn;
+
+use super::abstraction::Tree;
 
-#[derive(Clone)]
 pub struct Users {
-    pub(super) userid_password: sled::Tree,
-    pub(super) userid_displayname: sled::Tree,
-    pub(super) userid_avatarurl: sled::Tree,
-    pub(super) userdeviceid_token: sled::Tree,
-    pub(super) userdeviceid_metadata: sled::Tree, // This is also used to check if a device exists
-    pub(super) token_userdeviceid: sled::Tree,
+    pub(super) userid_password: Arc<dyn Tree>,
+    pub(super) userid_displayname: Arc<dyn Tree>,
+    pub(super) userid_avatarurl: Arc<dyn Tree>,
+    pub(super) userid_blurhash: Arc<dyn Tree>,
+    pub(super) userdeviceid_token: Arc<dyn Tree>,
+    pub(super) userdeviceid_metadata: Arc<dyn Tree>, // This is also used to check if a device exists
+    pub(super) userid_devicelistversion: Arc<dyn Tree>, // DevicelistVersion = u64
+    pub(super) token_userdeviceid: Arc<dyn Tree>,
 
-    pub(super) onetimekeyid_onetimekeys: sled::Tree, // OneTimeKeyId = UserId + DeviceKeyId
-    pub(super) userid_lastonetimekeyupdate: sled::Tree, // LastOneTimeKeyUpdate = Count
-    pub(super) keychangeid_userid: sled::Tree,       // KeyChangeId = UserId/RoomId + Count
-    pub(super) keyid_key: sled::Tree,                // KeyId = UserId + KeyId (depends on key type)
-    pub(super) userid_masterkeyid: sled::Tree,
-    pub(super) userid_selfsigningkeyid: sled::Tree,
-    pub(super) userid_usersigningkeyid: sled::Tree,
+    pub(super) onetimekeyid_onetimekeys: Arc<dyn Tree>, // OneTimeKeyId = UserId + DeviceKeyId
+    pub(super) userid_lastonetimekeyupdate: Arc<dyn Tree>, // LastOneTimeKeyUpdate = Count
+    pub(super) keychangeid_userid: Arc<dyn Tree>,       // KeyChangeId = UserId/RoomId + Count
+    pub(super) keyid_key: Arc<dyn Tree>, // KeyId = UserId + KeyId (depends on key type)
+    pub(super) userid_masterkeyid: Arc<dyn Tree>,
+    pub(super) userid_selfsigningkeyid: Arc<dyn Tree>,
+    pub(super) userid_usersigningkeyid: Arc<dyn Tree>,
 
-    pub(super) todeviceid_events: sled::Tree, // ToDeviceId = UserId + DeviceId + Count
+    pub(super) todeviceid_events: Arc<dyn Tree>, // ToDeviceId = UserId + DeviceId + Count
 }
 
 impl Users {
     /// Check if a user has an account on this homeserver.
+    #[tracing::instrument(skip(self, user_id))]
     pub fn exists(&self, user_id: &UserId) -> Result<bool> {
-        Ok(self.userid_password.contains_key(user_id.to_string())?)
+        Ok(self.userid_password.get(user_id.as_bytes())?.is_some())
     }
 
     /// Check if account is deactivated
+    #[tracing::instrument(skip(self, user_id))]
     pub fn is_deactivated(&self, user_id: &UserId) -> Result<bool> {
         Ok(self
             .userid_password
-            .get(user_id.to_string())?
+            .get(user_id.as_bytes())?
             .ok_or(Error::BadRequest(
                 ErrorKind::InvalidParam,
                 "User does not exist.",
@@ -54,20 +54,23 @@ impl Users {
     }
 
     /// Create a new user account on this homeserver.
-    pub fn create(&self, user_id: &UserId, password: &str) -> Result<()> {
+    #[tracing::instrument(skip(self, user_id, password))]
+    pub fn create(&self, user_id: &UserId, password: Option<&str>) -> Result<()> {
         self.set_password(user_id, password)?;
         Ok(())
     }
 
     /// Returns the number of users registered on this server.
-    pub fn count(&self) -> usize {
-        self.userid_password.iter().count()
+    #[tracing::instrument(skip(self))]
+    pub fn count(&self) -> Result<usize> {
+        Ok(self.userid_password.iter().count())
     }
 
     /// Find out which user an access token belongs to.
+    #[tracing::instrument(skip(self, token))]
     pub fn find_from_token(&self, token: &str) -> Result<Option<(UserId, String)>> {
         self.token_userdeviceid
-            .get(token)?
+            .get(token.as_bytes())?
             .map_or(Ok(None), |bytes| {
                 let mut parts = bytes.split(|&b| b == 0xff);
                 let user_bytes = parts.next().ok_or_else(|| {
@@ -92,21 +95,21 @@ impl Users {
     }
 
     /// Returns an iterator over all users on this homeserver.
-    pub fn iter(&self) -> impl Iterator<Item = Result<UserId>> {
-        self.userid_password.iter().keys().map(|bytes| {
-            Ok(
-                UserId::try_from(utils::string_from_bytes(&bytes?).map_err(|_| {
-                    Error::bad_database("User ID in userid_password is invalid unicode.")
-                })?)
-                .map_err(|_| Error::bad_database("User ID in userid_password is invalid."))?,
-            )
+    #[tracing::instrument(skip(self))]
+    pub fn iter(&self) -> impl Iterator<Item = Result<UserId>> + '_ {
+        self.userid_password.iter().map(|(bytes, _)| {
+            UserId::try_from(utils::string_from_bytes(&bytes).map_err(|_| {
+                Error::bad_database("User ID in userid_password is invalid unicode.")
+            })?)
+            .map_err(|_| Error::bad_database("User ID in userid_password is invalid."))
         })
     }
 
     /// Returns the password hash for the given user.
+    #[tracing::instrument(skip(self, user_id))]
     pub fn password_hash(&self, user_id: &UserId) -> Result<Option<String>> {
         self.userid_password
-            .get(user_id.to_string())?
+            .get(user_id.as_bytes())?
             .map_or(Ok(None), |bytes| {
                 Ok(Some(utils::string_from_bytes(&bytes).map_err(|_| {
                     Error::bad_database("Password hash in db is not valid string.")
@@ -115,22 +118,30 @@ impl Users {
     }
 
     /// Hash and set the user's password to the Argon2 hash
-    pub fn set_password(&self, user_id: &UserId, password: &str) -> Result<()> {
-        if let Ok(hash) = utils::calculate_hash(&password) {
-            self.userid_password.insert(user_id.to_string(), &*hash)?;
-            Ok(())
+    #[tracing::instrument(skip(self, user_id, password))]
+    pub fn set_password(&self, user_id: &UserId, password: Option<&str>) -> Result<()> {
+        if let Some(password) = password {
+            if let Ok(hash) = utils::calculate_hash(&password) {
+                self.userid_password
+                    .insert(user_id.as_bytes(), hash.as_bytes())?;
+                Ok(())
+            } else {
+                Err(Error::BadRequest(
+                    ErrorKind::InvalidParam,
+                    "Password does not meet the requirements.",
+                ))
+            }
         } else {
-            Err(Error::BadRequest(
-                ErrorKind::InvalidParam,
-                "Password does not meet the requirements.",
-            ))
+            self.userid_password.insert(user_id.as_bytes(), b"")?;
+            Ok(())
         }
     }
 
     /// Returns the displayname of a user on this homeserver.
+    #[tracing::instrument(skip(self, user_id))]
     pub fn displayname(&self, user_id: &UserId) -> Result<Option<String>> {
         self.userid_displayname
-            .get(user_id.to_string())?
+            .get(user_id.as_bytes())?
             .map_or(Ok(None), |bytes| {
                 Ok(Some(utils::string_from_bytes(&bytes).map_err(|_| {
                     Error::bad_database("Displayname in db is invalid.")
@@ -139,41 +150,73 @@ impl Users {
     }
 
     /// Sets a new displayname or removes it if displayname is None. You still need to nofify all rooms of this change.
+    #[tracing::instrument(skip(self, user_id, displayname))]
     pub fn set_displayname(&self, user_id: &UserId, displayname: Option<String>) -> Result<()> {
         if let Some(displayname) = displayname {
             self.userid_displayname
-                .insert(user_id.to_string(), &*displayname)?;
+                .insert(user_id.as_bytes(), displayname.as_bytes())?;
         } else {
-            self.userid_displayname.remove(user_id.to_string())?;
+            self.userid_displayname.remove(user_id.as_bytes())?;
         }
 
         Ok(())
     }
 
-    /// Get a the avatar_url of a user.
-    pub fn avatar_url(&self, user_id: &UserId) -> Result<Option<String>> {
+    /// Get the avatar_url of a user.
+    #[tracing::instrument(skip(self, user_id))]
+    pub fn avatar_url(&self, user_id: &UserId) -> Result<Option<MxcUri>> {
         self.userid_avatarurl
-            .get(user_id.to_string())?
-            .map_or(Ok(None), |bytes| {
-                Ok(Some(utils::string_from_bytes(&bytes).map_err(|_| {
-                    Error::bad_database("Avatar URL in db is invalid.")
-                })?))
+            .get(user_id.as_bytes())?
+            .map(|bytes| {
+                let s = utils::string_from_bytes(&bytes)
+                    .map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?;
+                MxcUri::try_from(s).map_err(|_| Error::bad_database("Avatar URL in db is invalid."))
             })
+            .transpose()
     }
 
     /// Sets a new avatar_url or removes it if avatar_url is None.
-    pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option<String>) -> Result<()> {
+    #[tracing::instrument(skip(self, user_id, avatar_url))]
+    pub fn set_avatar_url(&self, user_id: &UserId, avatar_url: Option<MxcUri>) -> Result<()> {
         if let Some(avatar_url) = avatar_url {
             self.userid_avatarurl
-                .insert(user_id.to_string(), &*avatar_url)?;
+                .insert(user_id.as_bytes(), avatar_url.to_string().as_bytes())?;
         } else {
-            self.userid_avatarurl.remove(user_id.to_string())?;
+            self.userid_avatarurl.remove(user_id.as_bytes())?;
+        }
+
+        Ok(())
+    }
+
+    /// Get the blurhash of a user.
+    #[tracing::instrument(skip(self, user_id))]
+    pub fn blurhash(&self, user_id: &UserId) -> Result<Option<String>> {
+        self.userid_blurhash
+            .get(user_id.as_bytes())?
+            .map(|bytes| {
+                let s = utils::string_from_bytes(&bytes)
+                    .map_err(|_| Error::bad_database("Avatar URL in db is invalid."))?;
+
+                Ok(s)
+            })
+            .transpose()
+    }
+
+    /// Sets a new avatar_url or removes it if avatar_url is None.
+    #[tracing::instrument(skip(self, user_id, blurhash))]
+    pub fn set_blurhash(&self, user_id: &UserId, blurhash: Option<String>) -> Result<()> {
+        if let Some(blurhash) = blurhash {
+            self.userid_blurhash
+                .insert(user_id.as_bytes(), blurhash.as_bytes())?;
+        } else {
+            self.userid_blurhash.remove(user_id.as_bytes())?;
         }
 
         Ok(())
     }
 
     /// Adds a new device to a user.
+    #[tracing::instrument(skip(self, user_id, device_id, token, initial_device_display_name))]
     pub fn create_device(
         &self,
         user_id: &UserId,
@@ -184,20 +227,22 @@ impl Users {
         // This method should never be called for nonexistent users.
         assert!(self.exists(user_id)?);
 
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
+        self.userid_devicelistversion
+            .increment(user_id.as_bytes())?;
+
         self.userdeviceid_metadata.insert(
-            userdeviceid,
-            serde_json::to_string(&Device {
+            &userdeviceid,
+            &serde_json::to_vec(&Device {
                 device_id: device_id.into(),
                 display_name: initial_device_display_name,
                 last_seen_ip: None, // TODO
-                last_seen_ts: Some(SystemTime::now()),
+                last_seen_ts: Some(MilliSecondsSinceUnixEpoch::now()),
             })
-            .expect("Device::to_string never fails.")
-            .as_bytes(),
+            .expect("Device::to_string never fails."),
         )?;
 
         self.set_token(user_id, &device_id, token)?;
@@ -206,13 +251,15 @@ impl Users {
     }
 
     /// Removes a device from a user.
+    #[tracing::instrument(skip(self, user_id, device_id))]
     pub fn remove_device(&self, user_id: &UserId, device_id: &DeviceId) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
         // Remove tokens
-        if let Some(old_token) = self.userdeviceid_token.remove(&userdeviceid)? {
+        if let Some(old_token) = self.userdeviceid_token.get(&userdeviceid)? {
+            self.userdeviceid_token.remove(&userdeviceid)?;
             self.token_userdeviceid.remove(&old_token)?;
         }
 
@@ -220,28 +267,34 @@ impl Users {
         let mut prefix = userdeviceid.clone();
         prefix.push(0xff);
 
-        for key in self.todeviceid_events.scan_prefix(&prefix).keys() {
-            self.todeviceid_events.remove(key?)?;
+        for (key, _) in self.todeviceid_events.scan_prefix(prefix) {
+            self.todeviceid_events.remove(&key)?;
         }
 
         // TODO: Remove onetimekeys
 
+        self.userid_devicelistversion
+            .increment(user_id.as_bytes())?;
+
         self.userdeviceid_metadata.remove(&userdeviceid)?;
 
         Ok(())
     }
 
     /// Returns an iterator over all device ids of this user.
-    pub fn all_device_ids(&self, user_id: &UserId) -> impl Iterator<Item = Result<Box<DeviceId>>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+    #[tracing::instrument(skip(self, user_id))]
+    pub fn all_device_ids<'a>(
+        &'a self,
+        user_id: &UserId,
+    ) -> impl Iterator<Item = Result<Box<DeviceId>>> + 'a {
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         // All devices have metadata
         self.userdeviceid_metadata
             .scan_prefix(prefix)
-            .keys()
-            .map(|bytes| {
+            .map(|(bytes, _)| {
                 Ok(utils::string_from_bytes(
-                    &*bytes?
+                    &bytes
                         .rsplit(|&b| b == 0xff)
                         .next()
                         .ok_or_else(|| Error::bad_database("UserDevice ID in db is invalid."))?,
@@ -252,8 +305,9 @@ impl Users {
     }
 
     /// Replaces the access token of one device.
-    fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+    #[tracing::instrument(skip(self, user_id, device_id, token))]
+    pub fn set_token(&self, user_id: &UserId, device_id: &DeviceId, token: &str) -> Result<()> {
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
@@ -262,17 +316,27 @@ impl Users {
 
         // Remove old token
         if let Some(old_token) = self.userdeviceid_token.get(&userdeviceid)? {
-            self.token_userdeviceid.remove(old_token)?;
+            self.token_userdeviceid.remove(&old_token)?;
             // It will be removed from userdeviceid_token by the insert later
         }
 
         // Assign token to user device combination
-        self.userdeviceid_token.insert(&userdeviceid, &*token)?;
-        self.token_userdeviceid.insert(token, userdeviceid)?;
+        self.userdeviceid_token
+            .insert(&userdeviceid, token.as_bytes())?;
+        self.token_userdeviceid
+            .insert(token.as_bytes(), &userdeviceid)?;
 
         Ok(())
     }
 
+    #[tracing::instrument(skip(
+        self,
+        user_id,
+        device_id,
+        one_time_key_key,
+        one_time_key_value,
+        globals
+    ))]
     pub fn add_one_time_key(
         &self,
         user_id: &UserId,
@@ -281,7 +345,7 @@ impl Users {
         one_time_key_value: &OneTimeKey,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(device_id.as_bytes());
 
@@ -300,21 +364,19 @@ impl Users {
 
         self.onetimekeyid_onetimekeys.insert(
             &key,
-            &*serde_json::to_string(&one_time_key_value)
-                .expect("OneTimeKey::to_string always works"),
+            &serde_json::to_vec(&one_time_key_value).expect("OneTimeKey::to_vec always works"),
         )?;
 
-        self.userid_lastonetimekeyupdate.insert(
-            &user_id.to_string().as_bytes(),
-            &globals.next_count()?.to_be_bytes(),
-        )?;
+        self.userid_lastonetimekeyupdate
+            .insert(&user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
 
         Ok(())
     }
 
+    #[tracing::instrument(skip(self, user_id))]
     pub fn last_one_time_keys_update(&self, user_id: &UserId) -> Result<u64> {
         self.userid_lastonetimekeyupdate
-            .get(&user_id.to_string().as_bytes())?
+            .get(&user_id.as_bytes())?
             .map(|bytes| {
                 utils::u64_from_bytes(&bytes).map_err(|_| {
                     Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")
@@ -323,6 +385,7 @@ impl Users {
             .unwrap_or(Ok(0))
     }
 
+    #[tracing::instrument(skip(self, user_id, device_id, key_algorithm, globals))]
     pub fn take_one_time_key(
         &self,
         user_id: &UserId,
@@ -330,24 +393,21 @@ impl Users {
         key_algorithm: &DeviceKeyAlgorithm,
         globals: &super::globals::Globals,
     ) -> Result<Option<(DeviceKeyId, OneTimeKey)>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(device_id.as_bytes());
         prefix.push(0xff);
         prefix.push(b'"'); // Annoying quotation mark
-        prefix.extend_from_slice(key_algorithm.to_string().as_bytes());
+        prefix.extend_from_slice(key_algorithm.as_ref().as_bytes());
         prefix.push(b':');
 
-        self.userid_lastonetimekeyupdate.insert(
-            &user_id.to_string().as_bytes(),
-            &globals.next_count()?.to_be_bytes(),
-        )?;
+        self.userid_lastonetimekeyupdate
+            .insert(&user_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
 
         self.onetimekeyid_onetimekeys
-            .scan_prefix(&prefix)
+            .scan_prefix(prefix)
             .next()
-            .map(|r| {
-                let (key, value) = r?;
+            .map(|(key, value)| {
                 self.onetimekeyid_onetimekeys.remove(&key)?;
 
                 Ok((
@@ -365,32 +425,32 @@ impl Users {
             .transpose()
     }
 
+    #[tracing::instrument(skip(self, user_id, device_id))]
     pub fn count_one_time_keys(
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
     ) -> Result<BTreeMap<DeviceKeyAlgorithm, UInt>> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
         let mut counts = BTreeMap::new();
 
-        for algorithm in self
-            .onetimekeyid_onetimekeys
-            .scan_prefix(&userdeviceid)
-            .keys()
-            .map(|bytes| {
-                Ok::<_, Error>(
-                    serde_json::from_slice::<DeviceKeyId>(
-                        &*bytes?.rsplit(|&b| b == 0xff).next().ok_or_else(|| {
-                            Error::bad_database("OneTimeKey ID in db is invalid.")
-                        })?,
+        for algorithm in
+            self.onetimekeyid_onetimekeys
+                .scan_prefix(userdeviceid)
+                .map(|(bytes, _)| {
+                    Ok::<_, Error>(
+                        serde_json::from_slice::<DeviceKeyId>(
+                            &*bytes.rsplit(|&b| b == 0xff).next().ok_or_else(|| {
+                                Error::bad_database("OneTimeKey ID in db is invalid.")
+                            })?,
+                        )
+                        .map_err(|_| Error::bad_database("DeviceKeyId in db is invalid."))?
+                        .algorithm(),
                     )
-                    .map_err(|_| Error::bad_database("DeviceKeyId in db is invalid."))?
-                    .algorithm(),
-                )
-            })
+                })
         {
             *counts.entry(algorithm?).or_default() += UInt::from(1_u32);
         }
@@ -398,6 +458,7 @@ impl Users {
         Ok(counts)
     }
 
+    #[tracing::instrument(skip(self, user_id, device_id, device_keys, rooms, globals))]
     pub fn add_device_keys(
         &self,
         user_id: &UserId,
@@ -406,13 +467,13 @@ impl Users {
         rooms: &super::rooms::Rooms,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
         self.keyid_key.insert(
             &userdeviceid,
-            &*serde_json::to_string(&device_keys).expect("DeviceKeys::to_string always works"),
+            &serde_json::to_vec(&device_keys).expect("DeviceKeys::to_vec always works"),
         )?;
 
         self.mark_device_key_update(user_id, rooms, globals)?;
@@ -420,6 +481,14 @@ impl Users {
         Ok(())
     }
 
+    #[tracing::instrument(skip(
+        self,
+        master_key,
+        self_signing_key,
+        user_signing_key,
+        rooms,
+        globals
+    ))]
     pub fn add_cross_signing_keys(
         &self,
         user_id: &UserId,
@@ -431,7 +500,7 @@ impl Users {
     ) -> Result<()> {
         // TODO: Check signatures
 
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         // Master key
@@ -453,11 +522,11 @@ impl Users {
 
         self.keyid_key.insert(
             &master_key_key,
-            &*serde_json::to_string(&master_key).expect("CrossSigningKey::to_string always works"),
+            &serde_json::to_vec(&master_key).expect("CrossSigningKey::to_vec always works"),
         )?;
 
         self.userid_masterkeyid
-            .insert(&*user_id.to_string(), master_key_key)?;
+            .insert(user_id.as_bytes(), &master_key_key)?;
 
         // Self-signing key
         if let Some(self_signing_key) = self_signing_key {
@@ -479,12 +548,12 @@ impl Users {
 
             self.keyid_key.insert(
                 &self_signing_key_key,
-                &*serde_json::to_string(&self_signing_key)
-                    .expect("CrossSigningKey::to_string always works"),
+                &serde_json::to_vec(&self_signing_key)
+                    .expect("CrossSigningKey::to_vec always works"),
             )?;
 
             self.userid_selfsigningkeyid
-                .insert(&*user_id.to_string(), self_signing_key_key)?;
+                .insert(user_id.as_bytes(), &self_signing_key_key)?;
         }
 
         // User-signing key
@@ -507,12 +576,12 @@ impl Users {
 
             self.keyid_key.insert(
                 &user_signing_key_key,
-                &*serde_json::to_string(&user_signing_key)
-                    .expect("CrossSigningKey::to_string always works"),
+                &serde_json::to_vec(&user_signing_key)
+                    .expect("CrossSigningKey::to_vec always works"),
             )?;
 
             self.userid_usersigningkeyid
-                .insert(&*user_id.to_string(), user_signing_key_key)?;
+                .insert(user_id.as_bytes(), &user_signing_key_key)?;
         }
 
         self.mark_device_key_update(user_id, rooms, globals)?;
@@ -520,6 +589,7 @@ impl Users {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self, target_id, key_id, signature, sender_id, rooms, globals))]
     pub fn sign_key(
         &self,
         target_id: &UserId,
@@ -529,9 +599,9 @@ impl Users {
         rooms: &super::rooms::Rooms,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut key = target_id.to_string().as_bytes().to_vec();
+        let mut key = target_id.as_bytes().to_vec();
         key.push(0xff);
-        key.extend_from_slice(key_id.to_string().as_bytes());
+        key.extend_from_slice(key_id.as_bytes());
 
         let mut cross_signing_key =
             serde_json::from_slice::<serde_json::Value>(&self.keyid_key.get(&key)?.ok_or(
@@ -554,8 +624,7 @@ impl Users {
 
         self.keyid_key.insert(
             &key,
-            &*serde_json::to_string(&cross_signing_key)
-                .expect("CrossSigningKey::to_string always works"),
+            &serde_json::to_vec(&cross_signing_key).expect("CrossSigningKey::to_vec always works"),
         )?;
 
         // TODO: Should we notify about this change?
@@ -564,39 +633,46 @@ impl Users {
         Ok(())
     }
 
-    pub fn keys_changed(
-        &self,
+    #[tracing::instrument(skip(self, user_or_room_id, from, to))]
+    pub fn keys_changed<'a>(
+        &'a self,
         user_or_room_id: &str,
         from: u64,
         to: Option<u64>,
-    ) -> impl Iterator<Item = Result<UserId>> {
+    ) -> impl Iterator<Item = Result<UserId>> + 'a {
         let mut prefix = user_or_room_id.as_bytes().to_vec();
         prefix.push(0xff);
 
         let mut start = prefix.clone();
         start.extend_from_slice(&(from + 1).to_be_bytes());
 
-        let mut end = prefix.clone();
-        end.extend_from_slice(&to.unwrap_or(u64::MAX).to_be_bytes());
+        let to = to.unwrap_or(u64::MAX);
 
         self.keychangeid_userid
-            .range(start..end)
-            .filter_map(|r| r.ok())
-            .take_while(move |(k, _)| k.starts_with(&prefix))
+            .iter_from(&start, false)
+            .take_while(move |(k, _)| {
+                k.starts_with(&prefix)
+                    && if let Some(current) = k.splitn(2, |&b| b == 0xff).nth(1) {
+                        if let Ok(c) = utils::u64_from_bytes(current) {
+                            c <= to
+                        } else {
+                            warn!("BadDatabase: Could not parse keychangeid_userid bytes");
+                            false
+                        }
+                    } else {
+                        warn!("BadDatabase: Could not parse keychangeid_userid");
+                        false
+                    }
+            })
             .map(|(_, bytes)| {
-                Ok(
-                    UserId::try_from(utils::string_from_bytes(&bytes).map_err(|_| {
-                        Error::bad_database(
-                            "User ID in devicekeychangeid_userid is invalid unicode.",
-                        )
-                    })?)
-                    .map_err(|_| {
-                        Error::bad_database("User ID in devicekeychangeid_userid is invalid.")
-                    })?,
-                )
+                UserId::try_from(utils::string_from_bytes(&bytes).map_err(|_| {
+                    Error::bad_database("User ID in devicekeychangeid_userid is invalid unicode.")
+                })?)
+                .map_err(|_| Error::bad_database("User ID in devicekeychangeid_userid is invalid."))
             })
     }
 
+    #[tracing::instrument(skip(self, user_id, rooms, globals))]
     fn mark_device_key_update(
         &self,
         user_id: &UserId,
@@ -613,47 +689,48 @@ impl Users {
                 continue;
             }
 
-            let mut key = room_id.to_string().as_bytes().to_vec();
+            let mut key = room_id.as_bytes().to_vec();
             key.push(0xff);
             key.extend_from_slice(&count);
 
-            self.keychangeid_userid.insert(key, &*user_id.to_string())?;
+            self.keychangeid_userid.insert(&key, user_id.as_bytes())?;
         }
 
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(&count);
-        self.keychangeid_userid.insert(key, &*user_id.to_string())?;
+        self.keychangeid_userid.insert(&key, user_id.as_bytes())?;
 
         Ok(())
     }
 
+    #[tracing::instrument(skip(self, user_id, device_id))]
     pub fn get_device_keys(
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
     ) -> Result<Option<DeviceKeys>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(device_id.as_bytes());
 
-        self.keyid_key.get(key)?.map_or(Ok(None), |bytes| {
+        self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
             Ok(Some(serde_json::from_slice(&bytes).map_err(|_| {
                 Error::bad_database("DeviceKeys in db are invalid.")
             })?))
         })
     }
 
-    pub fn get_master_key(
+    #[tracing::instrument(skip(self, user_id, allowed_signatures))]
+    pub fn get_master_key<F: Fn(&UserId) -> bool>(
         &self,
         user_id: &UserId,
-        sender_id: &UserId,
+        allowed_signatures: F,
     ) -> Result<Option<CrossSigningKey>> {
-        // TODO: hide some signatures
         self.userid_masterkeyid
-            .get(user_id.to_string())?
+            .get(user_id.as_bytes())?
             .map_or(Ok(None), |key| {
-                self.keyid_key.get(key)?.map_or(Ok(None), |bytes| {
+                self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
                     let mut cross_signing_key = serde_json::from_slice::<CrossSigningKey>(&bytes)
                         .map_err(|_| {
                         Error::bad_database("CrossSigningKey in db is invalid.")
@@ -664,7 +741,7 @@ impl Users {
                     cross_signing_key.signatures = cross_signing_key
                         .signatures
                         .into_iter()
-                        .filter(|(user, _)| user == user_id || user == sender_id)
+                        .filter(|(user, _)| allowed_signatures(user))
                         .collect();
 
                     Ok(Some(cross_signing_key))
@@ -672,15 +749,16 @@ impl Users {
             })
     }
 
-    pub fn get_self_signing_key(
+    #[tracing::instrument(skip(self, user_id, allowed_signatures))]
+    pub fn get_self_signing_key<F: Fn(&UserId) -> bool>(
         &self,
         user_id: &UserId,
-        sender_id: &UserId,
+        allowed_signatures: F,
     ) -> Result<Option<CrossSigningKey>> {
         self.userid_selfsigningkeyid
-            .get(user_id.to_string())?
+            .get(user_id.as_bytes())?
             .map_or(Ok(None), |key| {
-                self.keyid_key.get(key)?.map_or(Ok(None), |bytes| {
+                self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
                     let mut cross_signing_key = serde_json::from_slice::<CrossSigningKey>(&bytes)
                         .map_err(|_| {
                         Error::bad_database("CrossSigningKey in db is invalid.")
@@ -691,7 +769,7 @@ impl Users {
                     cross_signing_key.signatures = cross_signing_key
                         .signatures
                         .into_iter()
-                        .filter(|(user, _)| user == user_id || user == sender_id)
+                        .filter(|(user, _)| user == user_id || allowed_signatures(user))
                         .collect();
 
                     Ok(Some(cross_signing_key))
@@ -699,11 +777,12 @@ impl Users {
             })
     }
 
+    #[tracing::instrument(skip(self, user_id))]
     pub fn get_user_signing_key(&self, user_id: &UserId) -> Result<Option<CrossSigningKey>> {
         self.userid_usersigningkeyid
-            .get(user_id.to_string())?
+            .get(user_id.as_bytes())?
             .map_or(Ok(None), |key| {
-                self.keyid_key.get(key)?.map_or(Ok(None), |bytes| {
+                self.keyid_key.get(&key)?.map_or(Ok(None), |bytes| {
                     Ok(Some(serde_json::from_slice(&bytes).map_err(|_| {
                         Error::bad_database("CrossSigningKey in db is invalid.")
                     })?))
@@ -711,34 +790,43 @@ impl Users {
             })
     }
 
+    #[tracing::instrument(skip(
+        self,
+        sender,
+        target_user_id,
+        target_device_id,
+        event_type,
+        content,
+        globals
+    ))]
     pub fn add_to_device_event(
         &self,
         sender: &UserId,
         target_user_id: &UserId,
         target_device_id: &DeviceId,
-        event_type: &EventType,
+        event_type: &str,
         content: serde_json::Value,
         globals: &super::globals::Globals,
     ) -> Result<()> {
-        let mut key = target_user_id.to_string().as_bytes().to_vec();
+        let mut key = target_user_id.as_bytes().to_vec();
         key.push(0xff);
         key.extend_from_slice(target_device_id.as_bytes());
         key.push(0xff);
         key.extend_from_slice(&globals.next_count()?.to_be_bytes());
 
         let mut json = serde_json::Map::new();
-        json.insert("type".to_owned(), event_type.to_string().into());
+        json.insert("type".to_owned(), event_type.to_owned().into());
         json.insert("sender".to_owned(), sender.to_string().into());
         json.insert("content".to_owned(), content);
 
-        self.todeviceid_events.insert(
-            &key,
-            &*serde_json::to_string(&json).expect("Map::to_string always works"),
-        )?;
+        let value = serde_json::to_vec(&json).expect("Map::to_vec always works");
+
+        self.todeviceid_events.insert(&key, &value)?;
 
         Ok(())
     }
 
+    #[tracing::instrument(skip(self, user_id, device_id))]
     pub fn get_to_device_events(
         &self,
         user_id: &UserId,
@@ -746,14 +834,14 @@ impl Users {
     ) -> Result<Vec<Raw<AnyToDeviceEvent>>> {
         let mut events = Vec::new();
 
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(device_id.as_bytes());
         prefix.push(0xff);
 
-        for value in self.todeviceid_events.scan_prefix(&prefix).values() {
+        for (_, value) in self.todeviceid_events.scan_prefix(prefix) {
             events.push(
-                serde_json::from_slice(&*value?)
+                serde_json::from_slice(&value)
                     .map_err(|_| Error::bad_database("Event in todeviceid_events is invalid."))?,
             );
         }
@@ -761,13 +849,14 @@ impl Users {
         Ok(events)
     }
 
+    #[tracing::instrument(skip(self, user_id, device_id, until))]
     pub fn remove_to_device_events(
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
         until: u64,
     ) -> Result<()> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);
         prefix.extend_from_slice(device_id.as_bytes());
         prefix.push(0xff);
@@ -777,10 +866,9 @@ impl Users {
 
         for (key, _) in self
             .todeviceid_events
-            .range(&*prefix..=&*last)
-            .keys()
-            .map(|key| {
-                let key = key?;
+            .iter_from(&last, true) // this includes last
+            .take_while(move |(k, _)| k.starts_with(&prefix))
+            .map(|(key, _)| {
                 Ok::<_, Error>((
                     key.clone(),
                     utils::u64_from_bytes(&key[key.len() - mem::size_of::<u64>()..key.len()])
@@ -790,42 +878,45 @@ impl Users {
             .filter_map(|r| r.ok())
             .take_while(|&(_, count)| count <= until)
         {
-            self.todeviceid_events.remove(key)?;
+            self.todeviceid_events.remove(&key)?;
         }
 
         Ok(())
     }
 
+    #[tracing::instrument(skip(self, user_id, device_id, device))]
     pub fn update_device_metadata(
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
         device: &Device,
     ) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
         // Only existing devices should be able to call this.
         assert!(self.userdeviceid_metadata.get(&userdeviceid)?.is_some());
 
+        self.userid_devicelistversion
+            .increment(user_id.as_bytes())?;
+
         self.userdeviceid_metadata.insert(
-            userdeviceid,
-            serde_json::to_string(device)
-                .expect("Device::to_string always works")
-                .as_bytes(),
+            &userdeviceid,
+            &serde_json::to_vec(device).expect("Device::to_string always works"),
         )?;
 
         Ok(())
     }
 
     /// Get device metadata.
+    #[tracing::instrument(skip(self, user_id, device_id))]
     pub fn get_device_metadata(
         &self,
         user_id: &UserId,
         device_id: &DeviceId,
     ) -> Result<Option<Device>> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
+        let mut userdeviceid = user_id.as_bytes().to_vec();
         userdeviceid.push(0xff);
         userdeviceid.extend_from_slice(device_id.as_bytes());
 
@@ -838,21 +929,35 @@ impl Users {
             })
     }
 
-    pub fn all_devices_metadata(&self, user_id: &UserId) -> impl Iterator<Item = Result<Device>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+    #[tracing::instrument(skip(self, user_id))]
+    pub fn get_devicelist_version(&self, user_id: &UserId) -> Result<Option<u64>> {
+        self.userid_devicelistversion
+            .get(user_id.as_bytes())?
+            .map_or(Ok(None), |bytes| {
+                utils::u64_from_bytes(&bytes)
+                    .map_err(|_| Error::bad_database("Invalid devicelistversion in db."))
+                    .map(Some)
+            })
+    }
+
+    #[tracing::instrument(skip(self, user_id))]
+    pub fn all_devices_metadata<'a>(
+        &'a self,
+        user_id: &UserId,
+    ) -> impl Iterator<Item = Result<Device>> + 'a {
+        let mut key = user_id.as_bytes().to_vec();
         key.push(0xff);
 
         self.userdeviceid_metadata
             .scan_prefix(key)
-            .values()
-            .map(|bytes| {
-                Ok(serde_json::from_slice::<Device>(&bytes?).map_err(|_| {
-                    Error::bad_database("Device in userdeviceid_metadata is invalid.")
-                })?)
+            .map(|(_, bytes)| {
+                serde_json::from_slice::<Device>(&bytes)
+                    .map_err(|_| Error::bad_database("Device in userdeviceid_metadata is invalid."))
             })
     }
 
     /// Deactivate account
+    #[tracing::instrument(skip(self, user_id))]
     pub fn deactivate_account(&self, user_id: &UserId) -> Result<()> {
         // Remove all associated devices
         for device_id in self.all_device_ids(user_id) {
@@ -862,7 +967,7 @@ impl Users {
         // Set the password to "" to indicate a deactivated account. Hashes will never result in an
         // empty string, so the user will not be able to log in again. Systems like changing the
         // password without logging in should check if the account is deactivated.
-        self.userid_password.insert(user_id.to_string(), "")?;
+        self.userid_password.insert(user_id.as_bytes(), &[])?;
 
         // TODO: Unhook 3PID
         Ok(())

src/error.rs

@@ -1,13 +1,12 @@
-use std::{collections::HashMap, sync::RwLock, time::Duration, time::Instant};
-
-use log::error;
 use ruma::{
-    api::client::{error::ErrorKind, r0::uiaa::UiaaInfo},
-    events::room::message,
+    api::client::{
+        error::{Error as RumaError, ErrorKind},
+        r0::uiaa::UiaaInfo,
+    },
+    ServerName,
 };
 use thiserror::Error;
-
-use crate::{database::admin::AdminCommand, Database};
+use tracing::warn;
 
 #[cfg(feature = "conduit_bin")]
 use {
@@ -17,29 +16,47 @@ use {
         response::{self, Responder},
         Request,
     },
-    ruma::api::client::{error::Error as RumaError, r0::uiaa::UiaaResponse},
+    ruma::api::client::r0::uiaa::UiaaResponse,
+    tracing::error,
 };
 
 pub type Result<T> = std::result::Result<T, Error>;
 
 #[derive(Error, Debug)]
 pub enum Error {
-    #[error("There was a problem with the connection to the database.")]
+    #[cfg(feature = "sled")]
+    #[error("There was a problem with the connection to the sled database.")]
     SledError {
         #[from]
         source: sled::Error,
     },
+    #[cfg(feature = "sqlite")]
+    #[error("There was a problem with the connection to the sqlite database: {source}")]
+    SqliteError {
+        #[from]
+        source: rusqlite::Error,
+    },
+    #[cfg(feature = "heed")]
+    #[error("There was a problem with the connection to the heed database: {error}")]
+    HeedError { error: String },
     #[error("Could not generate an image.")]
     ImageError {
         #[from]
         source: image::error::ImageError,
     },
-    #[error("Could not connect to server.")]
+    #[error("Could not connect to server: {source}")]
     ReqwestError {
         #[from]
         source: reqwest::Error,
     },
     #[error("{0}")]
+    FederationError(Box<ServerName>, RumaError),
+    #[error("Could not do this io: {source}")]
+    IoError {
+        #[from]
+        source: std::io::Error,
+    },
+    #[error("{0}")]
     BadServerResponse(&'static str),
     #[error("{0}")]
     BadConfig(&'static str),
@@ -48,7 +65,6 @@ pub enum Error {
     BadDatabase(&'static str),
     #[error("uiaa")]
     Uiaa(UiaaInfo),
-
     #[error("{0}: {1}")]
     BadRequest(ErrorKind, &'static str),
     #[error("{0}")]
@@ -67,14 +83,16 @@ impl Error {
     }
 }
 
-#[cfg(feature = "conduit_bin")]
-impl<'r, 'o> Responder<'r, 'o> for Error
-where
-    'o: 'r,
-{
-    fn respond_to(self, r: &'r Request<'_>) -> response::Result<'o> {
-        if let Self::Uiaa(uiaainfo) = &self {
-            return RumaResponse::from(UiaaResponse::AuthResponse(uiaainfo.clone())).respond_to(r);
+impl Error {
+    pub fn to_response(&self) -> RumaResponse<UiaaResponse> {
+        if let Self::Uiaa(uiaainfo) = self {
+            return RumaResponse(UiaaResponse::AuthResponse(uiaainfo.clone()));
+        }
+
+        if let Self::FederationError(origin, error) = self {
+            let mut error = error.clone();
+            error.message = format!("Answer from {}: {}", origin, error.message);
+            return RumaResponse(UiaaResponse::MatrixError(error));
         }
 
         let message = format!("{}", self);
@@ -99,54 +117,22 @@ where
             _ => (Unknown, StatusCode::INTERNAL_SERVER_ERROR),
         };
 
-        RumaResponse::from(RumaError {
+        warn!("{}: {}", status_code, message);
+
+        RumaResponse(UiaaResponse::MatrixError(RumaError {
             kind,
             message,
             status_code,
-        })
-        .respond_to(r)
+        }))
     }
 }
 
-pub struct ConduitLogger {
-    pub db: Database,
-    pub last_logs: RwLock<HashMap<String, Instant>>,
-}
-
-impl log::Log for ConduitLogger {
-    fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool {
-        true
+#[cfg(feature = "conduit_bin")]
+impl<'r, 'o> Responder<'r, 'o> for Error
+where
+    'o: 'r,
+{
+    fn respond_to(self, r: &'r Request<'_>) -> response::Result<'o> {
+        self.to_response().respond_to(r)
     }
-
-    fn log(&self, record: &log::Record<'_>) {
-        let output = format!("{} - {}", record.level(), record.args());
-
-        println!("{}", output);
-
-        if self.enabled(record.metadata())
-            && record
-                .module_path()
-                .map_or(false, |path| path.starts_with("conduit::"))
-        {
-            if self
-                .last_logs
-                .read()
-                .unwrap()
-                .get(&output)
-                .map_or(false, |i| i.elapsed() < Duration::from_secs(60 * 30))
-            {
-                return;
-            }
-
-            if let Ok(mut_last_logs) = &mut self.last_logs.try_write() {
-                mut_last_logs.insert(output.clone(), Instant::now());
-            }
-
-            self.db.admin.send(AdminCommand::SendTextMessage(
-                message::TextMessageEventContent::plain(output),
-            ));
-        }
-    }
-
-    fn flush(&self) {}
 }

src/lib.rs

@@ -1,16 +1,19 @@
+#![allow(clippy::suspicious_else_formatting)]
+#![deny(clippy::dbg_macro)]
+
+pub mod appservice_server;
 pub mod client_server;
 mod database;
 mod error;
 mod pdu;
-mod push_rules;
 mod ruma_wrapper;
 pub mod server_server;
 mod utils;
 
-pub use database::Database;
+pub use database::{Config, Database};
 pub use error::{Error, Result};
 pub use pdu::PduEvent;
-pub use rocket::Config;
+pub use rocket::Config as RocketConfig;
 pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse};
 use std::ops::Deref;
 

src/main.rs

@@ -1,29 +1,42 @@
 #![warn(rust_2018_idioms)]
+#![allow(clippy::suspicious_else_formatting)]
+#![deny(clippy::dbg_macro)]
 
+pub mod appservice_server;
 pub mod client_server;
 pub mod server_server;
 
 mod database;
 mod error;
 mod pdu;
-mod push_rules;
 mod ruma_wrapper;
 mod utils;
 
+use std::sync::Arc;
+
+use database::Config;
 pub use database::Database;
-pub use error::{ConduitLogger, Error, Result};
-use log::LevelFilter;
+pub use error::{Error, Result};
+use opentelemetry::trace::Tracer;
 pub use pdu::PduEvent;
 pub use rocket::State;
+use ruma::api::client::error::ErrorKind;
 pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse};
 
-use rocket::{fairing::AdHoc, routes};
+use rocket::{
+    catch, catchers,
+    figment::{
+        providers::{Env, Format, Toml},
+        Figment,
+    },
+    routes, Request,
+};
+use tokio::sync::RwLock;
+use tracing_subscriber::{prelude::*, EnvFilter};
 
-fn setup_rocket() -> rocket::Rocket {
-    // Force log level off, so we can use our own logger
-    std::env::set_var("ROCKET_LOG_LEVEL", "off");
-
-    rocket::ignite()
+fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<rocket::Build> {
+    rocket::custom(config)
+        .manage(data)
         .mount(
             "/",
             routes![
@@ -37,21 +50,31 @@ fn setup_rocket() -> rocket::Rocket {
                 client_server::logout_all_route,
                 client_server::change_password_route,
                 client_server::deactivate_route,
+                client_server::third_party_route,
                 client_server::get_capabilities_route,
                 client_server::get_pushrules_all_route,
                 client_server::set_pushrule_route,
+                client_server::get_pushrule_route,
                 client_server::set_pushrule_enabled_route,
+                client_server::get_pushrule_enabled_route,
+                client_server::get_pushrule_actions_route,
+                client_server::set_pushrule_actions_route,
+                client_server::delete_pushrule_route,
                 client_server::get_room_event_route,
+                client_server::get_room_aliases_route,
                 client_server::get_filter_route,
                 client_server::create_filter_route,
                 client_server::set_global_account_data_route,
+                client_server::set_room_account_data_route,
                 client_server::get_global_account_data_route,
+                client_server::get_room_account_data_route,
                 client_server::set_displayname_route,
                 client_server::get_displayname_route,
                 client_server::set_avatar_url_route,
                 client_server::get_avatar_url_route,
                 client_server::get_profile_route,
                 client_server::set_presence_route,
+                client_server::get_presence_route,
                 client_server::upload_keys_route,
                 client_server::get_keys_route,
                 client_server::claim_keys_route,
@@ -69,6 +92,7 @@ fn setup_rocket() -> rocket::Rocket {
                 client_server::get_backup_key_sessions_route,
                 client_server::get_backup_keys_route,
                 client_server::set_read_marker_route,
+                client_server::create_receipt_route,
                 client_server::create_typing_event_route,
                 client_server::create_room_route,
                 client_server::redact_event_route,
@@ -122,6 +146,7 @@ fn setup_rocket() -> rocket::Rocket {
                 client_server::get_key_changes_route,
                 client_server::get_pushers_route,
                 client_server::set_pushers_route,
+                // client_server::third_party_route,
                 client_server::upgrade_room_route,
                 server_server::get_server_version_route,
                 server_server::get_server_keys_route,
@@ -129,29 +154,158 @@ fn setup_rocket() -> rocket::Rocket {
                 server_server::get_public_rooms_route,
                 server_server::get_public_rooms_filtered_route,
                 server_server::send_transaction_message_route,
+                server_server::get_event_route,
                 server_server::get_missing_events_route,
+                server_server::get_event_authorization_route,
+                server_server::get_room_state_route,
+                server_server::get_room_state_ids_route,
+                server_server::create_join_event_template_route,
+                server_server::create_join_event_route,
+                server_server::create_invite_route,
+                server_server::get_devices_route,
+                server_server::get_room_information_route,
                 server_server::get_profile_information_route,
+                server_server::get_keys_route,
+                server_server::claim_keys_route,
+            ],
+        )
+        .register(
+            "/",
+            catchers![
+                not_found_catcher,
+                forbidden_catcher,
+                unknown_token_catcher,
+                missing_token_catcher,
+                bad_json_catcher
             ],
         )
-        .attach(AdHoc::on_attach("Config", |rocket| async {
-            let data =
-                Database::load_or_create(rocket.figment().extract().expect("config is valid"))
-                    .await
-                    .expect("config is valid");
-
-            data.sending.start_handler(&data.globals, &data.rooms);
-            log::set_boxed_logger(Box::new(ConduitLogger {
-                db: data.clone(),
-                last_logs: Default::default(),
-            }))
-            .unwrap();
-            log::set_max_level(LevelFilter::Info);
-
-            Ok(rocket.manage(data))
-        }))
 }
 
 #[rocket::main]
 async fn main() {
-    setup_rocket().launch().await.unwrap();
+    // Force log level off, so we can use our own logger
+    std::env::set_var("CONDUIT_LOG_LEVEL", "off");
+
+    let raw_config =
+        Figment::from(default_config())
+            .merge(
+                Toml::file(Env::var("CONDUIT_CONFIG").expect(
+                    "The CONDUIT_CONFIG env var needs to be set. Example: /etc/conduit.toml",
+                ))
+                .nested(),
+            )
+            .merge(Env::prefixed("CONDUIT_").global());
+
+    std::env::set_var("RUST_LOG", "warn");
+
+    let config = raw_config
+        .extract::<Config>()
+        .expect("It looks like your config is invalid. Please take a look at the error");
+
+    let start = async {
+        config.warn_deprecated();
+
+        let db = Database::load_or_create(&config)
+            .await
+            .expect("config is valid");
+
+        let rocket = setup_rocket(raw_config, Arc::clone(&db))
+            .ignite()
+            .await
+            .unwrap();
+
+        Database::start_on_shutdown_tasks(db, rocket.shutdown()).await;
+
+        rocket.launch().await.unwrap();
+    };
+
+    if config.allow_jaeger {
+        let tracer = opentelemetry_jaeger::new_pipeline()
+            .with_service_name("conduit")
+            .install_simple()
+            .unwrap();
+
+        let span = tracer.start("conduit");
+        start.await;
+        drop(span);
+    } else {
+        std::env::set_var("RUST_LOG", &config.log);
+
+        let registry = tracing_subscriber::Registry::default();
+        if config.tracing_flame {
+            let (flame_layer, _guard) =
+                tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap();
+            let flame_layer = flame_layer.with_empty_samples(false);
+
+            let filter_layer = EnvFilter::new("trace,h2=off");
+
+            let subscriber = registry.with(filter_layer).with(flame_layer);
+            tracing::subscriber::set_global_default(subscriber).unwrap();
+            start.await;
+        } else {
+            let fmt_layer = tracing_subscriber::fmt::Layer::new();
+            let filter_layer = EnvFilter::try_from_default_env()
+                .or_else(|_| EnvFilter::try_new("info"))
+                .unwrap();
+
+            let subscriber = registry.with(filter_layer).with(fmt_layer);
+            tracing::subscriber::set_global_default(subscriber).unwrap();
+            start.await;
+        }
+    }
+}
+
+#[catch(404)]
+fn not_found_catcher(_: &Request<'_>) -> String {
+    "404 Not Found".to_owned()
+}
+
+#[catch(580)]
+fn forbidden_catcher() -> Result<()> {
+    Err(Error::BadRequest(ErrorKind::Forbidden, "Forbidden."))
+}
+
+#[catch(581)]
+fn unknown_token_catcher() -> Result<()> {
+    Err(Error::BadRequest(
+        ErrorKind::UnknownToken { soft_logout: false },
+        "Unknown token.",
+    ))
+}
+
+#[catch(582)]
+fn missing_token_catcher() -> Result<()> {
+    Err(Error::BadRequest(ErrorKind::MissingToken, "Missing token."))
+}
+
+#[catch(583)]
+fn bad_json_catcher() -> Result<()> {
+    Err(Error::BadRequest(ErrorKind::BadJson, "Bad json."))
+}
+
+fn default_config() -> rocket::Config {
+    let mut config = rocket::Config::release_default();
+
+    {
+        let mut shutdown = &mut config.shutdown;
+
+        #[cfg(unix)]
+        {
+            use rocket::config::Sig;
+
+            shutdown.signals.insert(Sig::Term);
+            shutdown.signals.insert(Sig::Int);
+        }
+
+        // Once shutdown is triggered, this is the amount of seconds before rocket
+        // will forcefully start shutting down connections, this gives enough time to /sync
+        // requests and the like (which havent gotten the memo, somehow) to still complete gracefully.
+        shutdown.grace = 35;
+
+        // After the grace period, rocket starts shutting down connections, and waits at least this
+        // many seconds before forcefully shutting all of them down.
+        shutdown.mercy = 10;
+    }
+
+    config
 }

src/pdu.rs

@@ -1,23 +1,20 @@
 use crate::Error;
-use js_int::UInt;
 use ruma::{
     events::{
-        pdu::EventHash, room::member::MemberEventContent, AnyEvent, AnyRoomEvent, AnyStateEvent,
-        AnyStrippedStateEvent, AnySyncRoomEvent, AnySyncStateEvent, EventType, StateEvent,
+        pdu::EventHash, room::member::MemberEventContent, AnyEphemeralRoomEvent,
+        AnyInitialStateEvent, AnyRoomEvent, AnyStateEvent, AnyStrippedStateEvent, AnySyncRoomEvent,
+        AnySyncStateEvent, EventType, StateEvent,
     },
-    serde::{to_canonical_value, CanonicalJsonObject, CanonicalJsonValue, Raw},
-    EventId, RoomId, RoomVersionId, ServerName, ServerSigningKeyId, UserId,
+    serde::{CanonicalJsonObject, CanonicalJsonValue, Raw},
+    state_res, EventId, MilliSecondsSinceUnixEpoch, RoomId, RoomVersionId, ServerName,
+    ServerSigningKeyId, UInt, UserId,
 };
 use serde::{Deserialize, Serialize};
 use serde_json::json;
-use std::{
-    collections::BTreeMap,
-    convert::{TryFrom, TryInto},
-    sync::Arc,
-    time::UNIX_EPOCH,
-};
+use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom};
+use tracing::error;
 
-#[derive(Deserialize, Serialize, Debug)]
+#[derive(Clone, Deserialize, Serialize, Debug)]
 pub struct PduEvent {
     pub event_id: EventId,
     pub room_id: RoomId,
@@ -33,13 +30,14 @@ pub struct PduEvent {
     pub auth_events: Vec<EventId>,
     #[serde(skip_serializing_if = "Option::is_none")]
     pub redacts: Option<EventId>,
-    #[serde(default, skip_serializing_if = "serde_json::Map::is_empty")]
-    pub unsigned: serde_json::Map<String, serde_json::Value>,
+    #[serde(default, skip_serializing_if = "BTreeMap::is_empty")]
+    pub unsigned: BTreeMap<String, serde_json::Value>,
     pub hashes: EventHash,
     pub signatures: BTreeMap<Box<ServerName>, BTreeMap<ServerSigningKeyId, String>>,
 }
 
 impl PduEvent {
+    #[tracing::instrument(skip(self))]
     pub fn redact(&mut self, reason: &PduEvent) -> crate::Result<()> {
         self.unsigned.clear();
 
@@ -76,9 +74,7 @@ impl PduEvent {
 
         self.unsigned.insert(
             "redacted_because".to_owned(),
-            serde_json::to_string(reason)
-                .expect("PduEvent::to_string always works")
-                .into(),
+            serde_json::to_value(reason).expect("to_value(PduEvent) always works"),
         );
 
         self.content = new_content.into();
@@ -86,6 +82,7 @@ impl PduEvent {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn to_sync_room_event(&self) -> Raw<AnySyncRoomEvent> {
         let mut json = json!({
             "content": self.content,
@@ -107,7 +104,8 @@ impl PduEvent {
     }
 
     /// This only works for events that are also AnyRoomEvents.
-    pub fn to_any_event(&self) -> Raw<AnyEvent> {
+    #[tracing::instrument(skip(self))]
+    pub fn to_any_event(&self) -> Raw<AnyEphemeralRoomEvent> {
         let mut json = json!({
             "content": self.content,
             "type": self.kind,
@@ -128,6 +126,7 @@ impl PduEvent {
         serde_json::from_value(json).expect("Raw::from_value always works")
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn to_room_event(&self) -> Raw<AnyRoomEvent> {
         let mut json = json!({
             "content": self.content,
@@ -149,6 +148,7 @@ impl PduEvent {
         serde_json::from_value(json).expect("Raw::from_value always works")
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn to_state_event(&self) -> Raw<AnyStateEvent> {
         let json = json!({
             "content": self.content,
@@ -164,6 +164,7 @@ impl PduEvent {
         serde_json::from_value(json).expect("Raw::from_value always works")
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn to_sync_state_event(&self) -> Raw<AnySyncStateEvent> {
         let json = json!({
             "content": self.content,
@@ -178,6 +179,7 @@ impl PduEvent {
         serde_json::from_value(json).expect("Raw::from_value always works")
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn to_stripped_state_event(&self) -> Raw<AnyStrippedStateEvent> {
         let json = json!({
             "content": self.content,
@@ -189,6 +191,7 @@ impl PduEvent {
         serde_json::from_value(json).expect("Raw::from_value always works")
     }
 
+    #[tracing::instrument(skip(self))]
     pub fn to_member_event(&self) -> Raw<StateEvent<MemberEventContent>> {
         let json = json!({
             "content": self.content,
@@ -206,10 +209,14 @@ impl PduEvent {
     }
 
     /// This does not return a full `Pdu` it is only to satisfy ruma's types.
+    #[tracing::instrument]
     pub fn convert_to_outgoing_federation_event(
         mut pdu_json: CanonicalJsonObject,
     ) -> Raw<ruma::events::pdu::Pdu> {
-        if let Some(CanonicalJsonValue::Object(unsigned)) = pdu_json.get_mut("unsigned") {
+        if let Some(unsigned) = pdu_json
+            .get_mut("unsigned")
+            .and_then(|val| val.as_object_mut())
+        {
             unsigned.remove("transaction_id");
         }
 
@@ -227,86 +234,107 @@ impl PduEvent {
         )
         .expect("Raw::from_value always works")
     }
-}
 
-impl From<&state_res::StateEvent> for PduEvent {
-    fn from(pdu: &state_res::StateEvent) -> Self {
-        Self {
-            event_id: pdu.event_id(),
-            room_id: pdu.room_id().clone(),
-            sender: pdu.sender().clone(),
-            origin_server_ts: (pdu
-                .origin_server_ts()
-                .duration_since(UNIX_EPOCH)
-                .expect("time is valid")
-                .as_millis() as u64)
-                .try_into()
-                .expect("time is valid"),
-            kind: pdu.kind(),
-            content: pdu.content().clone(),
-            state_key: Some(pdu.state_key()),
-            prev_events: pdu.prev_event_ids(),
-            depth: *pdu.depth(),
-            auth_events: pdu.auth_events(),
-            redacts: pdu.redacts().cloned(),
-            unsigned: pdu.unsigned().clone().into_iter().collect(),
-            hashes: pdu.hashes().clone(),
-            signatures: pdu.signatures(),
-        }
+    pub fn from_id_val(
+        event_id: &EventId,
+        mut json: CanonicalJsonObject,
+    ) -> Result<Self, serde_json::Error> {
+        json.insert(
+            "event_id".to_string(),
+            CanonicalJsonValue::String(event_id.as_str().to_owned()),
+        );
+
+        serde_json::from_value(serde_json::to_value(json).expect("valid JSON"))
     }
 }
 
-impl PduEvent {
-    pub fn convert_for_state_res(&self) -> Arc<state_res::StateEvent> {
-        Arc::new(
-            // For consistency of eventId (just in case) we use the one
-            // generated by conduit for everything.
-            state_res::StateEvent::from_id_value(
-                self.event_id.clone(),
-                json!({
-                    "event_id": self.event_id,
-                    "room_id": self.room_id,
-                    "sender": self.sender,
-                    "origin_server_ts": self.origin_server_ts,
-                    "type": self.kind,
-                    "content": self.content,
-                    "state_key": self.state_key,
-                    "prev_events": self.prev_events,
-                    "depth": self.depth,
-                    "auth_events": self.auth_events,
-                    "redacts": self.redacts,
-                    "unsigned": self.unsigned,
-                    "hashes": self.hashes,
-                    "signatures": self.signatures,
-                }),
-            )
-            .expect("all conduit PDUs are state events"),
-        )
+impl state_res::Event for PduEvent {
+    fn event_id(&self) -> &EventId {
+        &self.event_id
+    }
+
+    fn room_id(&self) -> &RoomId {
+        &self.room_id
+    }
+
+    fn sender(&self) -> &UserId {
+        &self.sender
+    }
+    fn kind(&self) -> EventType {
+        self.kind.clone()
+    }
+
+    fn content(&self) -> serde_json::Value {
+        self.content.clone()
+    }
+    fn origin_server_ts(&self) -> MilliSecondsSinceUnixEpoch {
+        MilliSecondsSinceUnixEpoch(self.origin_server_ts)
+    }
+    fn state_key(&self) -> Option<String> {
+        self.state_key.clone()
+    }
+    fn prev_events(&self) -> Vec<EventId> {
+        self.prev_events.to_vec()
+    }
+    fn depth(&self) -> &UInt {
+        &self.depth
+    }
+    fn auth_events(&self) -> Vec<EventId> {
+        self.auth_events.to_vec()
+    }
+    fn redacts(&self) -> Option<&EventId> {
+        self.redacts.as_ref()
+    }
+    fn hashes(&self) -> &EventHash {
+        &self.hashes
+    }
+    fn signatures(&self) -> BTreeMap<Box<ServerName>, BTreeMap<ruma::ServerSigningKeyId, String>> {
+        self.signatures.clone()
+    }
+    fn unsigned(&self) -> &BTreeMap<String, serde_json::Value> {
+        &self.unsigned
+    }
+}
+
+// These impl's allow us to dedup state snapshots when resolving state
+// for incoming events (federation/send/{txn}).
+impl Eq for PduEvent {}
+impl PartialEq for PduEvent {
+    fn eq(&self, other: &Self) -> bool {
+        self.event_id == other.event_id
+    }
+}
+impl PartialOrd for PduEvent {
+    fn partial_cmp(&self, other: &Self) -> Option<Ordering> {
+        self.event_id.partial_cmp(&other.event_id)
+    }
+}
+impl Ord for PduEvent {
+    fn cmp(&self, other: &Self) -> Ordering {
+        self.event_id.cmp(&other.event_id)
     }
 }
 
 /// Generates a correct eventId for the incoming pdu.
 ///
-/// Returns a tuple of the new `EventId` and the PDU with the eventId inserted as a `serde_json::Value`.
-pub(crate) fn process_incoming_pdu(
+/// Returns a tuple of the new `EventId` and the PDU as a `BTreeMap<String, CanonicalJsonValue>`.
+pub(crate) fn gen_event_id_canonical_json(
     pdu: &Raw<ruma::events::pdu::Pdu>,
-) -> (EventId, CanonicalJsonObject) {
-    let mut value =
-        serde_json::from_str(pdu.json().get()).expect("A Raw<...> is always valid JSON");
+) -> crate::Result<(EventId, CanonicalJsonObject)> {
+    let value = serde_json::from_str(pdu.json().get()).map_err(|e| {
+        error!("{:?}: {:?}", pdu, e);
+        Error::BadServerResponse("Invalid PDU in server response")
+    })?;
 
     let event_id = EventId::try_from(&*format!(
         "${}",
+        // Anything higher than version3 behaves the same
         ruma::signatures::reference_hash(&value, &RoomVersionId::Version6)
             .expect("ruma can calculate reference hashes")
     ))
     .expect("ruma's reference hashes are valid event ids");
 
-    value.insert(
-        "event_id".to_owned(),
-        to_canonical_value(&event_id).expect("EventId is a valid CanonicalJsonValue"),
-    );
-
-    (event_id, value)
+    Ok((event_id, value))
 }
 
 /// Build the start of a PDU in order to add it to the `Database`.
@@ -315,7 +343,21 @@ pub struct PduBuilder {
     #[serde(rename = "type")]
     pub event_type: EventType,
     pub content: serde_json::Value,
-    pub unsigned: Option<serde_json::Map<String, serde_json::Value>>,
+    pub unsigned: Option<BTreeMap<String, serde_json::Value>>,
     pub state_key: Option<String>,
     pub redacts: Option<EventId>,
 }
+
+/// Direct conversion prevents loss of the empty `state_key` that ruma requires.
+impl From<AnyInitialStateEvent> for PduBuilder {
+    fn from(event: AnyInitialStateEvent) -> Self {
+        Self {
+            event_type: EventType::from(event.event_type()),
+            content: serde_json::value::to_value(event.content())
+                .expect("AnyStateEventContent came from JSON and can thus turn back into JSON."),
+            unsigned: None,
+            state_key: Some(event.state_key().to_owned()),
+            redacts: None,
+        }
+    }
+}

src/push_rules.rs

@@ -1,256 +0,0 @@
-use ruma::{
-    push::{
-        Action, ConditionalPushRule, ConditionalPushRuleInit, ContentPushRule, OverridePushRule,
-        PatternedPushRule, PatternedPushRuleInit, PushCondition, RoomMemberCountIs, Ruleset, Tweak,
-        UnderridePushRule,
-    },
-    UserId,
-};
-
-pub fn default_pushrules(user_id: &UserId) -> Ruleset {
-    let mut rules = Ruleset::default();
-
-    rules.add(ContentPushRule(contains_user_name_rule(&user_id)));
-
-    for rule in vec![
-        master_rule(),
-        suppress_notices_rule(),
-        invite_for_me_rule(),
-        member_event_rule(),
-        contains_display_name_rule(),
-        tombstone_rule(),
-        roomnotif_rule(),
-    ] {
-        rules.add(OverridePushRule(rule));
-    }
-
-    for rule in vec![
-        call_rule(),
-        encrypted_room_one_to_one_rule(),
-        room_one_to_one_rule(),
-        message_rule(),
-        encrypted_rule(),
-    ] {
-        rules.add(UnderridePushRule(rule));
-    }
-
-    rules
-}
-
-pub fn master_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::DontNotify],
-        default: true,
-        enabled: false,
-        rule_id: ".m.rule.master".to_owned(),
-        conditions: vec![],
-    }
-    .into()
-}
-
-pub fn suppress_notices_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::DontNotify],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.suppress_notices".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "content.msgtype".to_owned(),
-            pattern: "m.notice".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn invite_for_me_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(false)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.invite_for_me".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "content.membership".to_owned(),
-            pattern: "m.invite".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn member_event_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::DontNotify],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.member_event".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "content.membership".to_owned(),
-            pattern: "type".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn contains_display_name_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(true)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.contains_display_name".to_owned(),
-        conditions: vec![PushCondition::ContainsDisplayName],
-    }
-    .into()
-}
-
-pub fn tombstone_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::Notify, Action::SetTweak(Tweak::Highlight(true))],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.tombstone".to_owned(),
-        conditions: vec![
-            PushCondition::EventMatch {
-                key: "type".to_owned(),
-                pattern: "m.room.tombstone".to_owned(),
-            },
-            PushCondition::EventMatch {
-                key: "state_key".to_owned(),
-                pattern: "".to_owned(),
-            },
-        ],
-    }
-    .into()
-}
-
-pub fn roomnotif_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::Notify, Action::SetTweak(Tweak::Highlight(true))],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.roomnotif".to_owned(),
-        conditions: vec![
-            PushCondition::EventMatch {
-                key: "content.body".to_owned(),
-                pattern: "@room".to_owned(),
-            },
-            PushCondition::SenderNotificationPermission {
-                key: "room".to_owned(),
-            },
-        ],
-    }
-    .into()
-}
-
-pub fn contains_user_name_rule(user_id: &UserId) -> PatternedPushRule {
-    PatternedPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(true)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.contains_user_name".to_owned(),
-        pattern: user_id.localpart().to_owned(),
-    }
-    .into()
-}
-
-pub fn call_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("ring".to_owned())),
-            Action::SetTweak(Tweak::Highlight(false)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.call".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "type".to_owned(),
-            pattern: "m.call.invite".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn encrypted_room_one_to_one_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(false)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.encrypted_room_one_to_one".to_owned(),
-        conditions: vec![
-            PushCondition::RoomMemberCount {
-                is: RoomMemberCountIs::from(2_u32.into()..),
-            },
-            PushCondition::EventMatch {
-                key: "type".to_owned(),
-                pattern: "m.room.encrypted".to_owned(),
-            },
-        ],
-    }
-    .into()
-}
-
-pub fn room_one_to_one_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![
-            Action::Notify,
-            Action::SetTweak(Tweak::Sound("default".to_owned())),
-            Action::SetTweak(Tweak::Highlight(false)),
-        ],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.room_one_to_one".to_owned(),
-        conditions: vec![
-            PushCondition::RoomMemberCount {
-                is: RoomMemberCountIs::from(2_u32.into()..),
-            },
-            PushCondition::EventMatch {
-                key: "type".to_owned(),
-                pattern: "m.room.message".to_owned(),
-            },
-        ],
-    }
-    .into()
-}
-
-pub fn message_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::Notify, Action::SetTweak(Tweak::Highlight(false))],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.message".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "type".to_owned(),
-            pattern: "m.room.message".to_owned(),
-        }],
-    }
-    .into()
-}
-
-pub fn encrypted_rule() -> ConditionalPushRule {
-    ConditionalPushRuleInit {
-        actions: vec![Action::Notify, Action::SetTweak(Tweak::Highlight(false))],
-        default: true,
-        enabled: true,
-        rule_id: ".m.rule.encrypted".to_owned(),
-        conditions: vec![PushCondition::EventMatch {
-            key: "type".to_owned(),
-            pattern: "m.room.encrypted".to_owned(),
-        }],
-    }
-    .into()
-}

src/ruma_wrapper.rs

@@ -1,30 +1,28 @@
-use crate::Error;
+use crate::{database::DatabaseGuard, Error};
 use ruma::{
-    api::{AuthScheme, OutgoingRequest},
+    api::{client::r0::uiaa::UiaaResponse, OutgoingResponse},
     identifiers::{DeviceId, UserId},
-    Outgoing,
-};
-use std::{
-    convert::{TryFrom, TryInto},
-    ops::Deref,
+    signatures::CanonicalJsonValue,
+    Outgoing, ServerName,
 };
+use std::ops::Deref;
 
 #[cfg(feature = "conduit_bin")]
 use {
-    crate::utils,
-    log::warn,
+    crate::server_server,
     rocket::{
-        data::{
-            ByteUnit, Data, FromDataFuture, FromTransformedData, Transform, TransformFuture,
-            Transformed,
-        },
+        data::{self, ByteUnit, Data, FromData},
         http::Status,
         outcome::Outcome::*,
         response::{self, Responder},
         tokio::io::AsyncReadExt,
-        Request, State,
+        Request,
     },
+    ruma::api::{AuthScheme, IncomingRequest},
+    std::collections::BTreeMap,
+    std::convert::TryFrom,
     std::io::Cursor,
+    tracing::{debug, warn},
 };
 
 /// This struct converts rocket requests into ruma structs by converting them into http requests
@@ -33,94 +31,303 @@ pub struct Ruma<T: Outgoing> {
     pub body: T::Incoming,
     pub sender_user: Option<UserId>,
     pub sender_device: Option<Box<DeviceId>>,
-    pub json_body: Option<Box<serde_json::value::RawValue>>, // This is None when body is not a valid string
+    pub sender_servername: Option<Box<ServerName>>,
+    // This is None when body is not a valid string
+    pub json_body: Option<CanonicalJsonValue>,
+    pub from_appservice: bool,
 }
 
 #[cfg(feature = "conduit_bin")]
-impl<'a, T: Outgoing + OutgoingRequest> FromTransformedData<'a> for Ruma<T>
+#[rocket::async_trait]
+impl<'a, T: Outgoing> FromData<'a> for Ruma<T>
 where
-    <T as Outgoing>::Incoming: TryFrom<http::request::Request<std::vec::Vec<u8>>> + std::fmt::Debug,
-    <<T as Outgoing>::Incoming as std::convert::TryFrom<
-        http::request::Request<std::vec::Vec<u8>>,
-    >>::Error: std::fmt::Debug,
+    T::Incoming: IncomingRequest,
 {
-    type Error = (); // TODO: Better error handling
-    type Owned = Data;
-    type Borrowed = Self::Owned;
+    type Error = ();
 
-    fn transform<'r>(
-        _req: &'r Request<'_>,
-        data: Data,
-    ) -> TransformFuture<'r, Self::Owned, Self::Error> {
-        Box::pin(async move { Transform::Owned(Success(data)) })
-    }
-
-    fn from_data(
+    #[tracing::instrument(skip(request, data))]
+    async fn from_data(
         request: &'a Request<'_>,
-        outcome: Transformed<'a, Self>,
-    ) -> FromDataFuture<'a, Self, Self::Error> {
-        Box::pin(async move {
-            let data = rocket::try_outcome!(outcome.owned());
-            let db = request
-                .guard::<State<'_, crate::Database>>()
-                .await
-                .expect("database was loaded");
+        data: Data<'a>,
+    ) -> data::Outcome<'a, Self, Self::Error> {
+        let metadata = T::Incoming::METADATA;
+        let db = request
+            .guard::<DatabaseGuard>()
+            .await
+            .expect("database was loaded");
 
-            let (sender_user, sender_device) = match T::METADATA.authentication {
+        // Get token from header or query value
+        let token = request
+            .headers()
+            .get_one("Authorization")
+            .and_then(|s| s.get(7..)) // Split off "Bearer "
+            .or_else(|| request.query_value("access_token").and_then(|r| r.ok()));
+
+        let limit = db.globals.max_request_size();
+        let mut handle = data.open(ByteUnit::Byte(limit.into()));
+        let mut body = Vec::new();
+        handle.read_to_end(&mut body).await.unwrap();
+
+        let mut json_body = serde_json::from_slice::<CanonicalJsonValue>(&body).ok();
+
+        let (sender_user, sender_device, sender_servername, from_appservice) = if let Some((
+            _id,
+            registration,
+        )) = db
+            .appservice
+            .all()
+            .unwrap()
+            .iter()
+            .find(|(_id, registration)| {
+                registration
+                    .get("as_token")
+                    .and_then(|as_token| as_token.as_str())
+                    .map_or(false, |as_token| token.as_deref() == Some(as_token))
+            }) {
+            match metadata.authentication {
                 AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
-                    // Get token from header or query value
-                    let token = match request
-                        .headers()
-                        .get_one("Authorization")
-                        .map(|s| s[7..].to_owned()) // Split off "Bearer "
-                        .or_else(|| request.get_query_value("access_token").and_then(|r| r.ok()))
-                    {
-                        // TODO: M_MISSING_TOKEN
-                        None => return Failure((Status::Unauthorized, ())),
-                        Some(token) => token,
-                    };
+                    let user_id = request.query_value::<String>("user_id").map_or_else(
+                        || {
+                            UserId::parse_with_server_name(
+                                registration
+                                    .get("sender_localpart")
+                                    .unwrap()
+                                    .as_str()
+                                    .unwrap(),
+                                db.globals.server_name(),
+                            )
+                            .unwrap()
+                        },
+                        |string| {
+                            UserId::try_from(string.expect("parsing to string always works"))
+                                .unwrap()
+                        },
+                    );
 
-                    // Check if token is valid
-                    match db.users.find_from_token(&token).unwrap() {
-                        // TODO: M_UNKNOWN_TOKEN
-                        None => return Failure((Status::Unauthorized, ())),
-                        Some((user_id, device_id)) => (Some(user_id), Some(device_id.into())),
+                    if !db.users.exists(&user_id).unwrap() {
+                        // Forbidden
+                        return Failure((Status::new(580), ()));
+                    }
+
+                    // TODO: Check if appservice is allowed to be that user
+                    (Some(user_id), None, None, true)
+                }
+                AuthScheme::ServerSignatures => (None, None, None, true),
+                AuthScheme::None => (None, None, None, true),
+            }
+        } else {
+            match metadata.authentication {
+                AuthScheme::AccessToken | AuthScheme::QueryOnlyAccessToken => {
+                    if let Some(token) = token {
+                        match db.users.find_from_token(&token).unwrap() {
+                            // Unknown Token
+                            None => return Failure((Status::new(581), ())),
+                            Some((user_id, device_id)) => (
+                                Some(user_id),
+                                Some(Box::<DeviceId>::from(device_id)),
+                                None,
+                                false,
+                            ),
+                        }
+                    } else {
+                        // Missing Token
+                        return Failure((Status::new(582), ()));
                     }
                 }
-                _ => (None, None),
-            };
+                AuthScheme::ServerSignatures => {
+                    // Get origin from header
+                    let x_matrix = match request
+                        .headers()
+                        .get_one("Authorization")
+                        .and_then(|s| s.get(9..)) // Split off "X-Matrix " and parse the rest
+                        .map(|s| {
+                            s.split_terminator(',')
+                                .map(|field| {
+                                    let mut splits = field.splitn(2, '=');
+                                    (splits.next(), splits.next().map(|s| s.trim_matches('"')))
+                                })
+                                .collect::<BTreeMap<_, _>>()
+                        }) {
+                        Some(t) => t,
+                        None => {
+                            warn!("No Authorization header");
 
-            let mut http_request = http::Request::builder()
-                .uri(request.uri().to_string())
-                .method(&*request.method().to_string());
-            for header in request.headers().iter() {
-                http_request = http_request.header(header.name.as_str(), &*header.value);
+                            // Forbidden
+                            return Failure((Status::new(580), ()));
+                        }
+                    };
+
+                    let origin_str = match x_matrix.get(&Some("origin")) {
+                        Some(Some(o)) => *o,
+                        _ => {
+                            warn!("Invalid X-Matrix header origin field: {:?}", x_matrix);
+
+                            // Forbidden
+                            return Failure((Status::new(580), ()));
+                        }
+                    };
+
+                    let origin = match Box::<ServerName>::try_from(origin_str) {
+                        Ok(s) => s,
+                        _ => {
+                            warn!(
+                                "Invalid server name in X-Matrix header origin field: {:?}",
+                                x_matrix
+                            );
+
+                            // Forbidden
+                            return Failure((Status::new(580), ()));
+                        }
+                    };
+
+                    let key = match x_matrix.get(&Some("key")) {
+                        Some(Some(k)) => *k,
+                        _ => {
+                            warn!("Invalid X-Matrix header key field: {:?}", x_matrix);
+
+                            // Forbidden
+                            return Failure((Status::new(580), ()));
+                        }
+                    };
+
+                    let sig = match x_matrix.get(&Some("sig")) {
+                        Some(Some(s)) => *s,
+                        _ => {
+                            warn!("Invalid X-Matrix header sig field: {:?}", x_matrix);
+
+                            // Forbidden
+                            return Failure((Status::new(580), ()));
+                        }
+                    };
+
+                    let mut request_map = BTreeMap::<String, CanonicalJsonValue>::new();
+
+                    if let Some(json_body) = &json_body {
+                        request_map.insert("content".to_owned(), json_body.clone());
+                    };
+
+                    request_map.insert(
+                        "method".to_owned(),
+                        CanonicalJsonValue::String(request.method().to_string()),
+                    );
+                    request_map.insert(
+                        "uri".to_owned(),
+                        CanonicalJsonValue::String(request.uri().to_string()),
+                    );
+                    request_map.insert(
+                        "origin".to_owned(),
+                        CanonicalJsonValue::String(origin.as_str().to_owned()),
+                    );
+                    request_map.insert(
+                        "destination".to_owned(),
+                        CanonicalJsonValue::String(db.globals.server_name().as_str().to_owned()),
+                    );
+
+                    let mut origin_signatures = BTreeMap::new();
+                    origin_signatures
+                        .insert(key.to_owned(), CanonicalJsonValue::String(sig.to_owned()));
+
+                    let mut signatures = BTreeMap::new();
+                    signatures.insert(
+                        origin.as_str().to_owned(),
+                        CanonicalJsonValue::Object(origin_signatures),
+                    );
+
+                    request_map.insert(
+                        "signatures".to_owned(),
+                        CanonicalJsonValue::Object(signatures),
+                    );
+
+                    let keys =
+                        match server_server::fetch_signing_keys(&db, &origin, vec![key.to_owned()])
+                            .await
+                        {
+                            Ok(b) => b,
+                            Err(e) => {
+                                warn!("Failed to fetch signing keys: {}", e);
+
+                                // Forbidden
+                                return Failure((Status::new(580), ()));
+                            }
+                        };
+
+                    let mut pub_key_map = BTreeMap::new();
+                    pub_key_map.insert(origin.as_str().to_owned(), keys);
+
+                    match ruma::signatures::verify_json(&pub_key_map, &request_map) {
+                        Ok(()) => (None, None, Some(origin), false),
+                        Err(e) => {
+                            warn!(
+                                "Failed to verify json request from {}: {}\n{:?}",
+                                origin, e, request_map
+                            );
+
+                            if request.uri().to_string().contains('@') {
+                                warn!("Request uri contained '@' character. Make sure your reverse proxy gives Conduit the raw uri (apache: use nocanon)");
+                            }
+
+                            // Forbidden
+                            return Failure((Status::new(580), ()));
+                        }
+                    }
+                }
+                AuthScheme::None => (None, None, None, false),
             }
+        };
 
-            let limit = db.globals.max_request_size();
-            let mut handle = data.open(ByteUnit::Byte(limit.into()));
-            let mut body = Vec::new();
-            handle.read_to_end(&mut body).await.unwrap();
+        let mut http_request = http::Request::builder()
+            .uri(request.uri().to_string())
+            .method(&*request.method().to_string());
+        for header in request.headers().iter() {
+            http_request = http_request.header(header.name.as_str(), &*header.value);
+        }
 
-            let http_request = http_request.body(body.clone()).unwrap();
-            log::debug!("{:?}", http_request);
+        if let Some(json_body) = json_body.as_mut().and_then(|val| val.as_object_mut()) {
+            let user_id = sender_user.clone().unwrap_or_else(|| {
+                UserId::parse_with_server_name("", db.globals.server_name())
+                    .expect("we know this is valid")
+            });
 
-            match <T as Outgoing>::Incoming::try_from(http_request) {
-                Ok(t) => Success(Ruma {
-                    body: t,
-                    sender_user,
-                    sender_device,
-                    // TODO: Can we avoid parsing it again? (We only need this for append_pdu)
-                    json_body: utils::string_from_bytes(&body)
+            if let Some(CanonicalJsonValue::Object(initial_request)) = json_body
+                .get("auth")
+                .and_then(|auth| auth.as_object())
+                .and_then(|auth| auth.get("session"))
+                .and_then(|session| session.as_str())
+                .and_then(|session| {
+                    db.uiaa
+                        .get_uiaa_request(
+                            &user_id,
+                            &sender_device.clone().unwrap_or_else(|| "".into()),
+                            session,
+                        )
                         .ok()
-                        .and_then(|s| serde_json::value::RawValue::from_string(s).ok()),
-                }),
-                Err(e) => {
-                    warn!("{:?}", e);
-                    Failure((Status::BadRequest, ()))
+                        .flatten()
+                })
+            {
+                for (key, value) in initial_request {
+                    json_body.entry(key).or_insert(value);
                 }
             }
-        })
+            body = serde_json::to_vec(json_body).expect("value to bytes can't fail");
+        }
+
+        let http_request = http_request.body(&*body).unwrap();
+        debug!("{:?}", http_request);
+        match <T::Incoming as IncomingRequest>::try_from_http_request(http_request) {
+            Ok(t) => Success(Ruma {
+                body: t,
+                sender_user,
+                sender_device,
+                sender_servername,
+                from_appservice,
+                json_body,
+            }),
+            Err(e) => {
+                warn!("{:?}", e);
+                // Bad Json
+                Failure((Status::new(583), ()))
+            }
+        }
     }
 }
 
@@ -135,51 +342,60 @@ impl<T: Outgoing> Deref for Ruma<T> {
 /// This struct converts ruma responses into rocket http responses.
 pub type ConduitResult<T> = std::result::Result<RumaResponse<T>, Error>;
 
-pub struct RumaResponse<T: TryInto<http::Response<Vec<u8>>>>(pub T);
+pub fn response<T: OutgoingResponse>(response: RumaResponse<T>) -> response::Result<'static> {
+    let http_response = response
+        .0
+        .try_into_http_response::<Vec<u8>>()
+        .map_err(|_| Status::InternalServerError)?;
 
-impl<T: TryInto<http::Response<Vec<u8>>>> From<T> for RumaResponse<T> {
+    let mut response = rocket::response::Response::build();
+
+    let status = http_response.status();
+    response.status(Status::new(status.as_u16()));
+
+    for header in http_response.headers() {
+        response.raw_header(header.0.to_string(), header.1.to_str().unwrap().to_owned());
+    }
+
+    let http_body = http_response.into_body();
+
+    response.sized_body(http_body.len(), Cursor::new(http_body));
+
+    response.raw_header("Access-Control-Allow-Origin", "*");
+    response.raw_header(
+        "Access-Control-Allow-Methods",
+        "GET, POST, PUT, DELETE, OPTIONS",
+    );
+    response.raw_header(
+        "Access-Control-Allow-Headers",
+        "Origin, X-Requested-With, Content-Type, Accept, Authorization",
+    );
+    response.raw_header("Access-Control-Max-Age", "86400");
+    response.ok()
+}
+
+#[derive(Clone)]
+pub struct RumaResponse<T>(pub T);
+
+impl<T> From<T> for RumaResponse<T> {
     fn from(t: T) -> Self {
         Self(t)
     }
 }
 
+impl From<Error> for RumaResponse<UiaaResponse> {
+    fn from(t: Error) -> Self {
+        t.to_response()
+    }
+}
+
 #[cfg(feature = "conduit_bin")]
 impl<'r, 'o, T> Responder<'r, 'o> for RumaResponse<T>
 where
-    T: Send + TryInto<http::Response<Vec<u8>>>,
-    T::Error: Send,
     'o: 'r,
+    T: OutgoingResponse,
 {
     fn respond_to(self, _: &'r Request<'_>) -> response::Result<'o> {
-        let http_response: Result<http::Response<_>, _> = self.0.try_into();
-        match http_response {
-            Ok(http_response) => {
-                let mut response = rocket::response::Response::build();
-
-                let status = http_response.status();
-                response.raw_status(status.into(), "");
-
-                for header in http_response.headers() {
-                    response
-                        .raw_header(header.0.to_string(), header.1.to_str().unwrap().to_owned());
-                }
-
-                let http_body = http_response.into_body();
-
-                response.sized_body(http_body.len(), Cursor::new(http_body));
-
-                response.raw_header("Access-Control-Allow-Origin", "*");
-                response.raw_header(
-                    "Access-Control-Allow-Methods",
-                    "GET, POST, PUT, DELETE, OPTIONS",
-                );
-                response.raw_header(
-                    "Access-Control-Allow-Headers",
-                    "Origin, X-Requested-With, Content-Type, Accept, Authorization",
-                );
-                response.ok()
-            }
-            Err(_) => Err(Status::InternalServerError),
-        }
+        response(self)
     }
 }

src/utils.rs

@@ -2,13 +2,14 @@ use argon2::{Config, Variant};
 use cmp::Ordering;
 use rand::prelude::*;
 use ruma::serde::{try_from_json_map, CanonicalJsonError, CanonicalJsonObject};
-use sled::IVec;
 use std::{
     cmp,
     convert::TryInto,
+    str::FromStr,
     time::{SystemTime, UNIX_EPOCH},
 };
 
+#[tracing::instrument]
 pub fn millis_since_unix_epoch() -> u64 {
     SystemTime::now()
         .duration_since(UNIX_EPOCH)
@@ -28,37 +29,40 @@ pub fn increment(old: Option<&[u8]>) -> Option<Vec<u8>> {
     Some(number.to_be_bytes().to_vec())
 }
 
-pub fn generate_keypair(old: Option<&[u8]>) -> Option<Vec<u8>> {
-    Some(old.map(|s| s.to_vec()).unwrap_or_else(|| {
-        let mut value = random_string(8).as_bytes().to_vec();
-        value.push(0xff);
-        value.extend_from_slice(
-            &ruma::signatures::Ed25519KeyPair::generate()
-                .expect("Ed25519KeyPair generation always works (?)"),
-        );
-        value
-    }))
+pub fn generate_keypair() -> Vec<u8> {
+    let mut value = random_string(8).as_bytes().to_vec();
+    value.push(0xff);
+    value.extend_from_slice(
+        &ruma::signatures::Ed25519KeyPair::generate()
+            .expect("Ed25519KeyPair generation always works (?)"),
+    );
+    value
 }
 
 /// Parses the bytes into an u64.
+#[tracing::instrument(skip(bytes))]
 pub fn u64_from_bytes(bytes: &[u8]) -> Result<u64, std::array::TryFromSliceError> {
     let array: [u8; 8] = bytes.try_into()?;
     Ok(u64::from_be_bytes(array))
 }
 
 /// Parses the bytes into a string.
+#[tracing::instrument(skip(bytes))]
 pub fn string_from_bytes(bytes: &[u8]) -> Result<String, std::string::FromUtf8Error> {
     String::from_utf8(bytes.to_vec())
 }
 
+#[tracing::instrument(skip(length))]
 pub fn random_string(length: usize) -> String {
     thread_rng()
         .sample_iter(&rand::distributions::Alphanumeric)
         .take(length)
+        .map(char::from)
         .collect()
 }
 
 /// Calculate a new hash for the given password
+#[tracing::instrument(skip(password))]
 pub fn calculate_hash(password: &str) -> Result<String, argon2::Error> {
     let hashing_config = Config {
         variant: Variant::Argon2id,
@@ -69,36 +73,35 @@ pub fn calculate_hash(password: &str) -> Result<String, argon2::Error> {
     argon2::hash_encoded(password.as_bytes(), salt.as_bytes(), &hashing_config)
 }
 
+#[tracing::instrument(skip(iterators, check_order))]
 pub fn common_elements(
-    mut iterators: impl Iterator<Item = impl Iterator<Item = IVec>>,
-    check_order: impl Fn(&IVec, &IVec) -> Ordering,
-) -> Option<impl Iterator<Item = IVec>> {
+    mut iterators: impl Iterator<Item = impl Iterator<Item = Vec<u8>>>,
+    check_order: impl Fn(&[u8], &[u8]) -> Ordering,
+) -> Option<impl Iterator<Item = Vec<u8>>> {
     let first_iterator = iterators.next()?;
     let mut other_iterators = iterators.map(|i| i.peekable()).collect::<Vec<_>>();
 
     Some(first_iterator.filter(move |target| {
-        other_iterators
-            .iter_mut()
-            .map(|it| {
-                while let Some(element) = it.peek() {
-                    match check_order(element, target) {
-                        Ordering::Greater => return false, // We went too far
-                        Ordering::Equal => return true,    // Element is in both iters
-                        Ordering::Less => {
-                            // Keep searching
-                            it.next();
-                        }
+        other_iterators.iter_mut().all(|it| {
+            while let Some(element) = it.peek() {
+                match check_order(element, target) {
+                    Ordering::Greater => return false, // We went too far
+                    Ordering::Equal => return true,    // Element is in both iters
+                    Ordering::Less => {
+                        // Keep searching
+                        it.next();
                     }
                 }
-                false
-            })
-            .all(|b| b)
+            }
+            false
+        })
     }))
 }
 
 /// Fallible conversion from any value that implements `Serialize` to a `CanonicalJsonObject`.
 ///
 /// `value` must serialize to an `serde_json::Value::Object`.
+#[tracing::instrument(skip(value))]
 pub fn to_canonical_object<T: serde::Serialize>(
     value: T,
 ) -> Result<CanonicalJsonObject, CanonicalJsonError> {
@@ -111,3 +114,30 @@ pub fn to_canonical_object<T: serde::Serialize>(
         ))),
     }
 }
+
+#[tracing::instrument(skip(deserializer))]
+pub fn deserialize_from_str<
+    'de,
+    D: serde::de::Deserializer<'de>,
+    T: FromStr<Err = E>,
+    E: std::fmt::Display,
+>(
+    deserializer: D,
+) -> std::result::Result<T, D::Error> {
+    struct Visitor<T: FromStr<Err = E>, E>(std::marker::PhantomData<T>);
+    impl<'de, T: FromStr<Err = Err>, Err: std::fmt::Display> serde::de::Visitor<'de>
+        for Visitor<T, Err>
+    {
+        type Value = T;
+        fn expecting(&self, formatter: &mut std::fmt::Formatter<'_>) -> std::fmt::Result {
+            write!(formatter, "a parsable string")
+        }
+        fn visit_str<E>(self, v: &str) -> Result<Self::Value, E>
+        where
+            E: serde::de::Error,
+        {
+            v.parse().map_err(serde::de::Error::custom)
+        }
+    }
+    deserializer.deserialize_str(Visitor(std::marker::PhantomData))
+}

tests/Complement.Dockerfile

@@ -1,3 +1,4 @@
+# For use in our CI only. This requires a build artifact created by a previous run pipline stage to be placed in cached_target/release/conduit
 FROM valkum/docker-rust-ci:latest as builder
 WORKDIR /workdir
 
@@ -9,7 +10,9 @@ ARG SCCACHE_ENDPOINT
 ARG SCCACHE_S3_USE_SSL
 
 COPY . .
-RUN cargo build
+RUN mkdir -p target/release
+RUN test -e cached_target/release/conduit && cp cached_target/release/conduit target/release/conduit || cargo build --release
+
 
 FROM valkum/docker-rust-ci:latest
 WORKDIR /workdir
@@ -17,24 +20,30 @@ WORKDIR /workdir
 RUN curl -OL "https://github.com/caddyserver/caddy/releases/download/v2.2.1/caddy_2.2.1_linux_amd64.tar.gz"
 RUN tar xzf caddy_2.2.1_linux_amd64.tar.gz
 
-COPY --from=builder /workdir/target/debug/conduit /workdir/conduit
+COPY cached_target/release/conduit /workdir/conduit
+RUN chmod +x /workdir/conduit
+RUN chmod +x /workdir/caddy
 
-COPY Rocket-example.toml Rocket.toml
+COPY conduit-example.toml conduit.toml
 
 ENV SERVER_NAME=localhost
 ENV ROCKET_LOG=normal
+ENV CONDUIT_CONFIG=/workdir/conduit.toml
 
-RUN sed -i "s/port = 14004/port = 8008/g" Rocket.toml
-RUN echo "federation_enabled = true" >> Rocket.toml
+RUN sed -i "s/port = 6167/port = 8008/g" conduit.toml
+RUN echo "allow_federation = true" >> conduit.toml
+RUN echo "allow_encryption = true" >> conduit.toml
+RUN echo "allow_registration = true" >> conduit.toml
+RUN echo "log = \"info,rocket=info,_=off,sled=off\"" >> conduit.toml
+RUN sed -i "s/address = \"127.0.0.1\"/address = \"0.0.0.0\"/g" conduit.toml
 
 # Enabled Caddy auto cert generation for complement provided CA.
-RUN echo '{"apps":{"http":{"https_port":8448,"servers":{"srv0":{"listen":[":8448"],"routes":[{"match":[{"host":["your.server.name"]}],"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"localhost:8008"}]}]}]}],"terminal":true}],"tls_connection_policies": [{"match": {"sni": ["your.server.name"]}}]}}},"pki": {"certificate_authorities": {"local": {"name": "Complement CA","root": {"certificate": "/ca/ca.crt","private_key": "/ca/ca.key"},"intermediate": {"certificate": "/ca/ca.crt","private_key": "/ca/ca.key"}}}},"tls":{"automation":{"policies":[{"subjects":["your.server.name"],"issuer":{"module":"internal"},"on_demand":true},{"issuer":{"module":"internal", "ca": "local"}}]}}}}' > caddy.json 
+RUN echo '{"logging":{"logs":{"default":{"level":"WARN"}}}, "apps":{"http":{"https_port":8448,"servers":{"srv0":{"listen":[":8448"],"routes":[{"match":[{"host":["your.server.name"]}],"handle":[{"handler":"subroute","routes":[{"handle":[{"handler":"reverse_proxy","upstreams":[{"dial":"127.0.0.1:8008"}]}]}]}],"terminal":true}],"tls_connection_policies": [{"match": {"sni": ["your.server.name"]}}]}}},"pki": {"certificate_authorities": {"local": {"name": "Complement CA","root": {"certificate": "/ca/ca.crt","private_key": "/ca/ca.key"},"intermediate": {"certificate": "/ca/ca.crt","private_key": "/ca/ca.key"}}}},"tls":{"automation":{"policies":[{"subjects":["your.server.name"],"issuer":{"module":"internal"},"on_demand":true},{"issuer":{"module":"internal", "ca": "local"}}]}}}}' > caddy.json 
  
 EXPOSE 8008 8448
 
 CMD ([ -z "${COMPLEMENT_CA}" ] && echo "Error: Need Complement PKI support" && true) || \
-    sed -i "s/server_name = \"your.server.name\"/server_name = \"${SERVER_NAME}\"/g" Rocket.toml && \
+    sed -i "s/#server_name = \"your.server.name\"/server_name = \"${SERVER_NAME}\"/g" conduit.toml && \
     sed -i "s/your.server.name/${SERVER_NAME}/g" caddy.json && \
     /workdir/caddy start --config caddy.json > /dev/null && \
     /workdir/conduit
-    

tests/client-element-web/test-element-web-registration.js

@@ -0,0 +1,101 @@
+const puppeteer = require('puppeteer');
+
+run().then(() => console.log('Done')).catch(error => {
+    console.error("Registration test failed.");
+    console.error("There might be a screenshot of the failure in the artifacts.\n");
+    console.error(error);
+    process.exit(111);
+});
+
+async function run() {
+
+    const elementUrl = process.argv[process.argv.length - 2];
+    console.debug("Testing registration with ElementWeb hosted at "+ elementUrl);
+
+    const homeserverUrl = process.argv[process.argv.length - 1];
+    console.debug("Homeserver url: "+ homeserverUrl);
+
+    const username = "testuser" + String(Math.floor(Math.random() * 100000));
+    const password = "testpassword" + String(Math.floor(Math.random() * 100000));
+    console.debug("Testuser for this run:\n  User: " + username + "\n  Password: " + password);
+
+    const browser = await puppeteer.launch({
+        headless: true, args: [
+            "--no-sandbox"
+        ]
+    });
+
+    const page = await browser.newPage();
+    await page.goto(elementUrl);
+
+    await page.screenshot({ path: '01-element-web-opened.png' });
+
+    console.debug("Click [Create Account] button");
+    await page.waitForSelector('a.mx_ButtonCreateAccount');
+    await page.click('a.mx_ButtonCreateAccount');
+
+    await page.screenshot({ path: '02-clicked-create-account-button.png' });
+
+    // The webapp should have loaded right now, if anything takes more than 5 seconds, something probably broke
+    page.setDefaultTimeout(5000);
+
+    console.debug("Click [Edit] to switch homeserver");
+    await page.waitForSelector('div.mx_ServerPicker_change');
+    await page.click('div.mx_ServerPicker_change');
+
+    await page.screenshot({ path: '03-clicked-edit-homeserver-button.png' });
+
+    console.debug("Type in local homeserver url");
+    await page.waitForSelector('input#mx_homeserverInput');
+    await page.click('input#mx_homeserverInput');
+    await page.click('input#mx_homeserverInput');
+    await page.keyboard.type(homeserverUrl);
+
+    await page.screenshot({ path: '04-typed-in-homeserver.png' });
+
+    console.debug("[Continue] with changed homeserver");
+    await page.waitForSelector("div.mx_ServerPickerDialog_continue");
+    await page.click('div.mx_ServerPickerDialog_continue');
+
+    await page.screenshot({ path: '05-back-to-enter-user-credentials.png' });
+
+    console.debug("Type in username");
+    await page.waitForSelector("input#mx_RegistrationForm_username");
+    await page.click('input#mx_RegistrationForm_username');
+    await page.keyboard.type(username);
+
+    await page.screenshot({ path: '06-typed-in-username.png' });
+
+    console.debug("Type in password");
+    await page.waitForSelector("input#mx_RegistrationForm_password");
+    await page.click('input#mx_RegistrationForm_password');
+    await page.keyboard.type(password);
+
+    await page.screenshot({ path: '07-typed-in-password-once.png' });
+
+    console.debug("Type in password again");
+    await page.waitForSelector("input#mx_RegistrationForm_passwordConfirm");
+    await page.click('input#mx_RegistrationForm_passwordConfirm');
+    await page.keyboard.type(password);
+
+    await page.screenshot({ path: '08-typed-in-password-twice.png' });
+
+    console.debug("Click on [Register] to finish the account creation");
+    await page.waitForSelector("input.mx_Login_submit");
+    await page.click('input.mx_Login_submit');
+
+    await page.screenshot({ path: '09-clicked-on-register-button.png' });
+
+    // Waiting for the app to login can take some time, so be patient.
+    page.setDefaultTimeout(10000);
+
+    console.debug("Wait for chat window to show up");
+    await page.waitForSelector("div.mx_HomePage_default_buttons");
+    console.debug("Apparently the registration worked.");
+
+    await page.screenshot({ path: '10-logged-in-homescreen.png' });
+
+
+    // Close the browser and exit the script
+    await browser.close();
+}

tests/sytest/are-we-synapse-yet.py

@@ -11,7 +11,7 @@ import sys
 # The main complexity is grouping tests sensibly into features like 'Registration'
 # and 'Federation'. Then it just checks the ones which are passing and calculates
 # percentages for each group. Produces results like:
-#
+# 
 # Client-Server APIs: 29% (196/666 tests)
 # -------------------
 #   Registration             :  62% (20/32 tests)
@@ -28,11 +28,13 @@ import sys
 #    ✓ POST /register can create a user
 #    ✓ POST /register downcases capitals in usernames
 #    ...
-#
+# 
 # You can also tack `-v` on to see exactly which tests each category falls under.
 
 test_mappings = {
     "nsp": "Non-Spec API",
+    "unk": "Unknown API (no group specified)",
+    "app": "Application Services API",
     "f": "Federation", # flag to mark test involves federation
 
     "federation_apis": {
@@ -50,6 +52,7 @@ test_mappings = {
         "fpb": "Public Room API",
         "fdk": "Device Key APIs",
         "fed": "Federation API",
+		"fsd": "Send-to-Device APIs",
     },
 
     "client_apis": {
@@ -61,6 +64,8 @@ test_mappings = {
         "pro": "Profile",
         "dev": "Devices",
         "dvk": "Device Keys",
+        "dkb": "Device Key Backup",
+        "xsk": "Cross-signing Keys",
         "pre": "Presence",
         "crm": "Create Room",
         "syn": "Sync API",
@@ -98,7 +103,7 @@ test_mappings = {
         "adm": "Server Admin API",
         "ign": "Ignore Users",
         "udr": "User Directory APIs",
-        "app": "Application Services API",
+		"jso": "Enforced canonical JSON",
     },
 }
 
@@ -156,20 +161,22 @@ def print_stats(header_name, gid_to_tests, gid_to_name, verbose):
     total_tests = 0
     for gid, tests in gid_to_tests.items():
         group_total = len(tests)
+        if group_total == 0:
+            continue
         group_passing = 0
         test_names_and_marks = []
         for name, passing in tests.items():
             if passing:
                 group_passing += 1
             test_names_and_marks.append(f"{'✓' if passing else '×'} {name}")
-
+            
         total_tests += group_total
         total_passing += group_passing
         pct = "{0:.0f}%".format(group_passing/group_total * 100)
         line = "%s: %s (%d/%d tests)" % (gid_to_name[gid].ljust(25, ' '), pct.rjust(4, ' '), group_passing, group_total)
         subsections.append(line)
         subsection_test_names[line] = test_names_and_marks
-
+    
     pct = "{0:.0f}%".format(total_passing/total_tests * 100)
     print("%s: %s (%d/%d tests)" % (header_name, pct, total_passing, total_tests))
     print("-" * (len(header_name)+1))
@@ -186,7 +193,6 @@ def main(results_tap_path, verbose):
     test_name_to_group_id = {}
     fed_tests = set()
     client_tests = set()
-    groupless_tests = set()
     with open("./are-we-synapse-yet.list", "r") as f:
         for line in f.readlines():
             test_name = " ".join(line.split(" ")[1:]).strip()
@@ -212,8 +218,12 @@ def main(results_tap_path, verbose):
             #   test_name: OK
             # }
         },
+        "appservice": {
+            "app": {},
+        },
         "nonspec": {
-            "nsp": {}
+            "nsp": {},
+            "unk": {}
         },
     }
     with open(results_tap_path, "r") as f:
@@ -224,10 +234,11 @@ def main(results_tap_path, verbose):
             name = test_result["name"]
             group_id = test_name_to_group_id.get(name)
             if not group_id:
-                groupless_tests.add(name)
-                # raise Exception("The test '%s' doesn't have a group" % (name,))
+                summary["nonspec"]["unk"][name] = test_result["ok"]
             if group_id == "nsp":
                 summary["nonspec"]["nsp"][name] = test_result["ok"]
+            elif group_id == "app":
+                summary["appservice"]["app"][name] = test_result["ok"]
             elif group_id in test_mappings["federation_apis"]:
                 group = summary["federation"].get(group_id, {})
                 group[name] = test_result["ok"]
@@ -243,12 +254,7 @@ def main(results_tap_path, verbose):
     print_stats("Non-Spec APIs", summary["nonspec"], test_mappings, verbose)
     print_stats("Client-Server APIs", summary["client"], test_mappings["client_apis"], verbose)
     print_stats("Federation APIs", summary["federation"], test_mappings["federation_apis"], verbose)
-    if verbose:
-        print("The following tests don't have a group:")
-        for name in groupless_tests:
-            print("  %s" % (name,))
-    else:
-        print("%d tests don't have a group" % len(groupless_tests))
+    print_stats("Application Services APIs", summary["appservice"], test_mappings, verbose)
 
 
 
@@ -257,4 +263,4 @@ if __name__ == '__main__':
     parser.add_argument("tap_file", help="path to results.tap")
     parser.add_argument("-v", action="store_true", help="show individual test names in output")
     args = parser.parse_args()
-    main(args.tap_file, args.v)
+    main(args.tap_file, args.v)

tests/sytest/sytest-whitelist

@@ -1,71 +1,335 @@
+/event/ does not allow access to events before the user joined
+/event/ on joined room works
+/event/ on non world readable room does not work
 /joined_members return joined members
 /joined_rooms returns only joined rooms
+/whois
 3pid invite join valid signature but revoked keys are rejected
 3pid invite join valid signature but unreachable ID server are rejected
 3pid invite join with wrong but valid signature are rejected
+A change to displayname should appear in incremental /sync
+A full_state incremental update returns all state
+A full_state incremental update returns only recent timeline
+A message sent after an initial sync appears in the timeline of an incremental sync.
+A next_batch token can be used in the v1 messages API
+A pair of events which redact each other should be ignored
+A pair of servers can establish a join in a v2 room
+A prev_batch token can be used in the v1 messages API
+AS can create a user
+AS can create a user with an underscore
+AS can create a user with inhibit_login
+AS can set avatar for ghosted users
+AS can set displayname for ghosted users
+AS can't set displayname for random users
 AS cannot create users outside its own namespace
+AS user (not ghost) can join room without registering
+AS user (not ghost) can join room without registering, with user_id query param
+After changing password, a different session no longer works by default
+After changing password, can log in with new password
+After changing password, can't log in with old password
+After changing password, different sessions can optionally be kept
+After changing password, existing session still works
 After deactivating account, can't log in with an email
+After deactivating account, can't log in with password
 Alias creators can delete alias with no ops
 Alias creators can delete canonical alias with no ops
 Alternative server names do not cause a routing loop
+An event which redacts an event in a different room should be ignored
+An event which redacts itself should be ignored
+Asking for a remote rooms list, but supplying the local server's name, returns the local rooms list
+Backfill checks the events requested belong to the room
+Backfill works correctly with history visibility set to joined
+Backfilled events whose prev_events are in a different room do not allow cross-room back-pagination
+Banned servers cannot /event_auth
+Banned servers cannot /invite
+Banned servers cannot /make_join
+Banned servers cannot /make_leave
+Banned servers cannot /send_join
+Banned servers cannot /send_leave
+Banned servers cannot backfill
+Banned servers cannot get missing events
+Banned servers cannot get room state
+Banned servers cannot get room state ids
+Banned servers cannot send events
+Banned user is kicked and may not rejoin until unbanned
 Both GET and PUT work
+Can /sync newly created room
 Can add account data
+Can add account data to room
 Can add tag
+Can claim one time key using POST
+Can claim remote one time key using POST
 Can create filter
+Can deactivate account
 Can delete canonical alias
+Can download file 'ascii'
+Can download file 'name with spaces'
+Can download file 'name;with;semicolons'
+Can download filter
+Can download specifying a different ASCII file name
+Can download specifying a different Unicode file name
+Can download with Unicode file name locally
+Can download with Unicode file name over federation
+Can download without a file name locally
+Can download without a file name over federation
+Can forget room you've been kicked from
+Can get 'm.room.name' state for a departed room (SPEC-216)
+Can get account data without syncing
+Can get remote public room list
+Can get room account data without syncing
+Can get rooms/{roomId}/members
+Can get rooms/{roomId}/members for a departed room (SPEC-216)
+Can get rooms/{roomId}/state for a departed room (SPEC-216)
 Can invite users to invite-only rooms
 Can list tags for a room
 Can logout all devices
 Can logout current device
+Can paginate public room list
+Can pass a JSON filter as a query parameter
+Can query device keys using POST
+Can query remote device keys using POST
+Can query specific device keys using POST
 Can re-join room if re-invited
 Can read configuration endpoint
+Can receive redactions from regular users over federation in room version 1
+Can receive redactions from regular users over federation in room version 2
+Can receive redactions from regular users over federation in room version 3
+Can receive redactions from regular users over federation in room version 4
+Can receive redactions from regular users over federation in room version 5
+Can receive redactions from regular users over federation in room version 6
 Can recv a device message using /sync
+Can recv a device message using /sync 
+Can recv device messages over federation 
 Can recv device messages until they are acknowledged
+Can recv device messages until they are acknowledged 
+Can reject invites over federation for rooms with version 1
+Can reject invites over federation for rooms with version 2
+Can reject invites over federation for rooms with version 3
+Can reject invites over federation for rooms with version 4
+Can reject invites over federation for rooms with version 5
+Can reject invites over federation for rooms with version 6
 Can remove tag
+Can search public room list
 Can send a message directly to a device using PUT /sendToDevice
+Can send a message directly to a device using PUT /sendToDevice 
 Can send a to-device message to two users which both receive it using /sync
+Can send image in room message
 Can send messages with a wildcard device id
+Can send messages with a wildcard device id 
 Can send messages with a wildcard device id to two devices
+Can send messages with a wildcard device id to two devices 
 Can sync
+Can sync a joined room
+Can sync a room with a message with a transaction id
+Can sync a room with a single message
+Can upload device keys
 Can upload with ASCII file name
 Can upload with Unicode file name
 Can upload without a file name
+Can't deactivate account with wrong password
+Can't forget room you're still in
+Changes to state are included in an gapped incremental sync
+Changes to state are included in an incremental sync
 Changing the actions of an unknown default rule fails with 404
 Changing the actions of an unknown rule fails with 404
 Checking local federation server
+Creators can delete alias
 Current state appears in timeline in private history
 Current state appears in timeline in private history with many messages before
+DELETE /device/{deviceId}
+DELETE /device/{deviceId} requires UI auth user to match device owner
+DELETE /device/{deviceId} with no body gives a 401
 Deleted tags appear in an incremental v2 /sync
 Deleting a non-existent alias should return a 404
+Device list doesn't change if remote server is down
+Device messages over federation wake up /sync 
 Device messages wake up /sync
+Device messages wake up /sync 
 Device messages with the same txn_id are deduplicated
+Device messages with the same txn_id are deduplicated 
+Enabling an unknown default rule fails with 404
+Event size limits
+Event with an invalid signature in the send_join response should not cause room join to fail
 Events come down the correct room
+Events whose auth_events are in the wrong room do not mess up the room state
+Existing members see new members' join events
+Federation key API allows unsigned requests for keys
+Federation key API can act as a notary server via a GET request
+Federation key API can act as a notary server via a POST request
+Federation rejects inbound events where the prev_events cannot be found
+Fetching eventstream a second time doesn't yield the message again
+Forgetting room does not show up in v2 /sync
+Full state sync includes joined rooms
+GET /capabilities is present and well formed for registered user 
 GET /device/{deviceId}
 GET /device/{deviceId} gives a 404 for unknown devices
 GET /devices
+GET /directory/room/:room_alias yields room ID
+GET /events initially
 GET /events with negative 'limit'
 GET /events with non-numeric 'limit'
 GET /events with non-numeric 'timeout'
+GET /initialSync initially
 GET /joined_rooms lists newly-created room
 GET /login yields a set of flows
 GET /media/r0/download can fetch the value again
 GET /profile/:user_id/avatar_url publicly accessible
 GET /profile/:user_id/displayname publicly accessible
+GET /publicRooms includes avatar URLs
 GET /publicRooms lists newly-created room
+GET /publicRooms lists rooms
+GET /r0/capabilities is not public
 GET /register yields a set of flows
+GET /rooms/:room_id/joined_members fetches my membership
+GET /rooms/:room_id/messages returns a message
 GET /rooms/:room_id/state fetches entire room state
 GET /rooms/:room_id/state/m.room.member/:user_id fetches my membership
+GET /rooms/:room_id/state/m.room.member/:user_id?format=event fetches my membership event
+GET /rooms/:room_id/state/m.room.name gets name
+GET /rooms/:room_id/state/m.room.power_levels can fetch levels
+GET /rooms/:room_id/state/m.room.power_levels fetches powerlevels
+GET /rooms/:room_id/state/m.room.topic gets topic
+Get left notifs for other users in sync and /keys/changes when user leaves
+Getting messages going forward is limited for a departed room (SPEC-216)
 Getting push rules doesn't corrupt the cache SYN-390
+Getting state IDs checks the events requested belong to the room
+Getting state checks the events requested belong to the room
+Ghost user must register before joining room
+Guest non-joined user cannot call /events on default room
+Guest non-joined user cannot call /events on invited room
+Guest non-joined user cannot call /events on joined room
+Guest non-joined user cannot call /events on shared room
+Guest non-joined users can get individual state for world_readable rooms
+Guest non-joined users can get individual state for world_readable rooms after leaving
+Guest non-joined users can get state for world_readable rooms
+Guest non-joined users cannot room initalSync for non-world_readable rooms
+Guest non-joined users cannot send messages to guest_access rooms if not joined
+Guest user can set display names
+Guest user cannot call /events globally
+Guest user cannot upgrade other users
+Guest users can accept invites to private rooms over federation
+Guest users can join guest_access rooms
+Guest users can send messages to guest_access rooms if joined
+If a device list update goes missing, the server resyncs on the next one
+If remote user leaves room we no longer receive device updates
+If remote user leaves room, changes device and rejoins we see update in /keys/changes
+If remote user leaves room, changes device and rejoins we see update in sync
+Inbound /make_join rejects attempts to join rooms where all users have left
+Inbound /v1/make_join rejects remote attempts to join local users to rooms
+Inbound /v1/send_join rejects incorrectly-signed joins
+Inbound /v1/send_join rejects joins from other servers
+Inbound /v1/send_leave rejects leaves from other servers
+Inbound federation accepts a second soft-failed event
+Inbound federation accepts attempts to join v2 rooms from servers with support
+Inbound federation can backfill events
+Inbound federation can get public room list
+Inbound federation can get state for a room
+Inbound federation can get state_ids for a room
+Inbound federation can query profile data
+Inbound federation can query room alias directory
+Inbound federation can receive events
+Inbound federation can receive invites via v1 API
+Inbound federation can receive invites via v2 API
+Inbound federation can receive redacted events
+Inbound federation can receive v1 /send_join
+Inbound federation can receive v2 /send_join
+Inbound federation can return events
+Inbound federation can return missing events for invite visibility
+Inbound federation can return missing events for world_readable visibility
+Inbound federation correctly soft fails events
+Inbound federation of state requires event_id as a mandatory paramater
+Inbound federation of state_ids requires event_id as a mandatory paramater
+Inbound federation rejects attempts to join v1 rooms from servers without v1 support
+Inbound federation rejects attempts to join v2 rooms from servers lacking version support
+Inbound federation rejects attempts to join v2 rooms from servers only supporting v1
+Inbound federation rejects invite rejections which include invalid JSON for room version 6
+Inbound federation rejects invites which include invalid JSON for room version 6
+Inbound federation rejects receipts from wrong remote
+Inbound federation rejects remote attempts to join local users to rooms
+Inbound federation rejects remote attempts to kick local users to rooms
+Inbound federation rejects typing notifications from wrong remote
+Inbound: send_join rejects invalid JSON for room version 6
+Invalid JSON floats
+Invalid JSON integers
+Invalid JSON special values
+Invited user can reject invite
+Invited user can reject invite over federation
+Invited user can reject invite over federation for empty room
+Invited user can reject invite over federation several times
+Invited user can see room metadata
+Inviting an AS-hosted user asks the AS server
+Lazy loading parameters in the filter are strictly boolean
+Left rooms appear in the leave section of full state sync
+Local delete device changes appear in v2 /sync
+Local device key changes appear in /keys/changes
+Local device key changes appear in v2 /sync
+Local device key changes get to remote servers
+Local new device changes appear in v2 /sync
+Local non-members don't see posted message events
+Local room members can get room messages
+Local room members see posted message events
+Local update device changes appear in v2 /sync
+Local users can peek by room alias
+Local users can peek into world_readable rooms by room ID
+Message history can be paginated
+Message history can be paginated over federation
+Name/topic keys are correct
+New account data appears in incremental v2 /sync
+New read receipts appear in incremental v2 /sync
+New room members see their own join event
+New users appear in /keys/changes
+Newly banned rooms appear in the leave section of incremental sync
+Newly joined room is included in an incremental sync
+Newly joined room is included in an incremental sync after invite
+Newly left rooms appear in the leave section of gapped sync
+Newly left rooms appear in the leave section of incremental sync
 Newly updated tags appear in an incremental v2 /sync
+Non-numeric ports in server names are rejected
+Outbound federation can backfill events
+Outbound federation can query profile data
+Outbound federation can query room alias directory
+Outbound federation can query v1 /send_join
+Outbound federation can query v2 /send_join
+Outbound federation can request missing events
+Outbound federation can send events
+Outbound federation can send invites via v1 API
+Outbound federation can send invites via v2 API
+Outbound federation can send room-join requests
+Outbound federation correctly handles unsupported room versions
+Outbound federation passes make_join failures through to the client
+Outbound federation rejects backfill containing invalid JSON for events in room version 6
+Outbound federation rejects m.room.create events with an unknown room version
+Outbound federation rejects send_join responses with no m.room.create event
+Outbound federation sends receipts
+Outbound federation will ignore a missing event with bad JSON for room version 6
+POST /createRoom creates a room with the given version
+POST /createRoom ignores attempts to set the room version via creation_content
 POST /createRoom makes a private room
 POST /createRoom makes a private room with invites
+POST /createRoom makes a public room
+POST /createRoom makes a room with a name
+POST /createRoom makes a room with a topic
+POST /createRoom rejects attempts to create rooms with numeric versions
+POST /createRoom rejects attempts to create rooms with unknown versions
+POST /createRoom with creation content
+POST /join/:room_alias can join a room
+POST /join/:room_alias can join a room with custom content
 POST /join/:room_id can join a room
+POST /join/:room_id can join a room with custom content
 POST /login as non-existing user is rejected
 POST /login can log in as a user
 POST /login can log in as a user with just the local part of the id
 POST /login returns the same device_id as that in the request
 POST /login wrong password is rejected
 POST /media/r0/upload can create an upload
+POST /redact disallows redaction of event in different room
+POST /register allows registration of usernames with '-'
+POST /register allows registration of usernames with '.'
+POST /register allows registration of usernames with '/'
+POST /register allows registration of usernames with '3'
+POST /register allows registration of usernames with '='
+POST /register allows registration of usernames with '_'
+POST /register allows registration of usernames with 'q'
 POST /register can create a user
 POST /register downcases capitals in usernames
 POST /register rejects registration of usernames with '!'
@@ -88,41 +352,161 @@ POST /rooms/:room_id/ban can ban a user
 POST /rooms/:room_id/invite can send an invite
 POST /rooms/:room_id/join can join a room
 POST /rooms/:room_id/leave can leave a room
+POST /rooms/:room_id/read_markers can create read marker
+POST /rooms/:room_id/receipt can create receipts
+POST /rooms/:room_id/redact/:event_id as original message sender redacts message
+POST /rooms/:room_id/redact/:event_id as power user redacts message
+POST /rooms/:room_id/redact/:event_id as random user does not redact message
+POST /rooms/:room_id/send/:event_type sends a message
 POST /rooms/:room_id/state/m.room.name sets name
 POST /rooms/:room_id/state/m.room.topic sets topic
 POST /rooms/:room_id/upgrade can upgrade a room version
+POST rejects invalid utf-8 in JSON
 POSTed media can be thumbnailed
 PUT /device/{deviceId} gives a 404 for unknown devices
 PUT /device/{deviceId} updates device fields
 PUT /directory/room/:room_alias creates alias
 PUT /profile/:user_id/avatar_url sets my avatar
 PUT /profile/:user_id/displayname sets my name
+PUT /rooms/:room_id/send/:event_type/:txn_id deduplicates the same txn id
+PUT /rooms/:room_id/send/:event_type/:txn_id sends a message
 PUT /rooms/:room_id/state/m.room.power_levels can set levels
+PUT /rooms/:room_id/typing/:user_id sets typing notification
 PUT power_levels should not explode if the old power levels were empty
+Peeked rooms only turn up in the sync for the device who peeked them
+Previously left rooms don't appear in the leave section of sync
 Push rules come down in an initial /sync
 Read markers appear in incremental v2 /sync
 Read markers appear in initial v2 /sync
 Read markers can be updated
+Read receipts appear in initial v2 /sync
+Real non-joined user cannot call /events on default room
+Real non-joined user cannot call /events on invited room
+Real non-joined user cannot call /events on joined room
+Real non-joined user cannot call /events on shared room
+Real non-joined users can get individual state for world_readable rooms
+Real non-joined users can get individual state for world_readable rooms after leaving
+Real non-joined users can get state for world_readable rooms
+Real non-joined users cannot room initalSync for non-world_readable rooms
+Real non-joined users cannot send messages to guest_access rooms if not joined
+Receipts must be m.read
+Redaction of a redaction redacts the redaction reason
 Regular users can add and delete aliases in the default room configuration
+Regular users can add and delete aliases when m.room.aliases is restricted
+Regular users cannot create room aliases within the AS namespace
+Regular users cannot register within the AS namespace
+Remote media can be thumbnailed
+Remote room alias queries can handle Unicode
+Remote room members also see posted message events
+Remote room members can get room messages
+Remote user can backfill in a room with version 1
+Remote user can backfill in a room with version 2
+Remote user can backfill in a room with version 3
+Remote user can backfill in a room with version 4
+Remote user can backfill in a room with version 5
+Remote user can backfill in a room with version 6
+Remote users can join room by alias
+Remote users may not join unfederated rooms
+Request to logout with invalid an access token is rejected
+Request to logout without an access token is rejected
+Room aliases can contain Unicode
 Room creation reports m.room.create to myself
 Room creation reports m.room.member to myself
+Room members can join a room with an overridden displayname
+Room members can override their displayname on a room-specific basis
+Room state at a rejected message event is the same as its predecessor
+Room state at a rejected state event is the same as its predecessor
 Rooms a user is invited to appear in an incremental sync
 Rooms a user is invited to appear in an initial sync
+Rooms can be created with an initial invite list (SYN-205)
+Server correctly handles incoming m.device_list_update
+Server correctly handles transactions that break edu limits
+Server correctly resyncs when client query keys and there is no remote cache
+Server correctly resyncs when server leaves and rejoins a room
+Server rejects invalid JSON in a version 6 room
 Setting room topic reports m.room.topic to myself
+Should not be able to take over the room by pretending there is no PL event
 Should reject keys claiming to belong to a different user
+State from remote users is included in the state in the initial sync
+State from remote users is included in the timeline in an incremental sync
+State is included in the timeline in the initial sync
+Sync can be polled for updates
+Sync is woken up for leaves
+Syncing a new room with a large timeline limit isn't limited
 Tags appear in an initial v2 /sync
 Trying to get push rules with unknown rule_id fails with 404
+Typing can be explicitly stopped
 Typing events appear in gapped sync
 Typing events appear in incremental sync
 Typing events appear in initial sync
+Typing notification sent to local room members
+Typing notifications also sent to remote room members
+Typing notifications don't leak
 Uninvited users cannot join the room
+Unprivileged users can set m.room.topic if it only needs level 0
 User appears in user directory
+User can create and send/receive messages in a room with version 1
+User can create and send/receive messages in a room with version 2
+User can create and send/receive messages in a room with version 3
+User can create and send/receive messages in a room with version 4
+User can create and send/receive messages in a room with version 5
+User can create and send/receive messages in a room with version 6
+User can invite local user to room with version 1
+User can invite local user to room with version 2
+User can invite local user to room with version 3
+User can invite local user to room with version 4
+User can invite local user to room with version 5
+User can invite local user to room with version 6
+User can invite remote user to room with version 1
+User can invite remote user to room with version 2
+User can invite remote user to room with version 3
+User can invite remote user to room with version 4
+User can invite remote user to room with version 5
+User can invite remote user to room with version 6
 User directory correctly update on display name change
 User in dir while user still shares private rooms
 User in shared private room does appear in user directory
 User is offline if they set_presence=offline in their sync
+User signups are forbidden from starting with '_'
+Users can't delete other's aliases
+Users cannot invite a user that is already in the room
+Users cannot invite themselves to a room
+Users cannot kick users from a room they are not in
+Users cannot kick users who have already left a room
+Users cannot set ban powerlevel higher than their own
+Users cannot set kick powerlevel higher than their own
+Users cannot set notifications powerlevel higher than their own
+Users cannot set redact powerlevel higher than their own
+Users receive device_list updates for their own devices
 Users with sufficient power-level can delete other's aliases
 Version responds 200 OK with valid structure
+We can't peek into rooms with invited history_visibility
+We can't peek into rooms with joined history_visibility
+We can't peek into rooms with shared history_visibility
+We don't send redundant membership state across incremental syncs by default
 We should see our own leave event when rejecting an invite, even if history_visibility is restricted (riot-web/3462)
+We should see our own leave event, even if history_visibility is restricted (SYN-662)
+Wildcard device messages over federation wake up /sync 
 Wildcard device messages wake up /sync
+Wildcard device messages wake up /sync 
+avatar_url updates affect room member events
+displayname updates affect room member events
+local user can join room with version 1
+local user can join room with version 2
+local user can join room with version 3
+local user can join room with version 4
+local user can join room with version 5
+local user can join room with version 6
+m.room.history_visibility == "joined" allows/forbids appropriately for Guest users
+m.room.history_visibility == "joined" allows/forbids appropriately for Real users
+m.room.history_visibility == "world_readable" allows/forbids appropriately for Guest users
+m.room.history_visibility == "world_readable" allows/forbids appropriately for Real users
 query for user with no keys returns empty key dict
+remote user can join room with version 1
+remote user can join room with version 2
+remote user can join room with version 3
+remote user can join room with version 4
+remote user can join room with version 5
+remote user can join room with version 6
+setting 'm.room.name' respects room powerlevel
+setting 'm.room.power_levels' respects room powerlevel

tests/test-config.toml

@@ -0,0 +1,15 @@
+[global]
+
+# Server runs in same container as tests do, so localhost is fine
+server_name = "localhost"
+
+# With a bit of luck /tmp is a RAM disk, so that the file system does not become the bottleneck while testing
+database_path = "/tmp"
+
+# All the other settings are left at their defaults:
+port = 6167
+max_request_size = 20_000_000
+allow_registration = true
+trusted_servers = ["matrix.org"]
+address = "127.0.0.1"
+proxy = "none"