docs: WIP: Adjust suggested value ranges to Timo's suggestion and when to change them

Signed-off-by: Jonas Zohren <git-pbkyr@jzohren.de>
docs: WIP: Added some suggested range for numeric values
2021-09-15 12:45:13 +02:00 · 2021-09-15 12:38:17 +02:00 · 2021-09-15 12:21:43 +02:00 · 2021-09-15 11:16:19 +02:00 · 2021-09-15 11:16:19 +02:00 · 2021-09-15 11:16:18 +02:00
108 changed files with 16328 additions and 6307 deletions
@@ -0,0 +1,11 @@
+---
+name: "Issue with / Feature Request for Conduit"
+about: "Please file issues on GitLab: https://gitlab.com/famedly/conduit/-/issues/new"
+title: "CLOSE ME"
+---
+
+
+
+**⚠️ Conduit development does not happen on GitHub. Issues opened here will not be addressed** 
+
+Please open issues on GitLab: https://gitlab.com/famedly/conduit/-/issues/new
@@ -1,5 +1,65 @@
-/target
-**/*.rs.bk
+# CMake
+cmake-build-*/

+# IntelliJ
+.idea/
+out/
+*.iml
+modules.xml
+*.ipr
+
+# mpeltonen/sbt-idea plugin
+.idea_modules/
+
+# Linux backup files
+*~
+
+# temporary files which can be created if a process still has a handle open of a deleted file
+.fuse_hidden*
+
+# KDE directory preferences
+.directory
+
+# Linux trash folder which might appear on any partition or disk
+.Trash-*
+
+# .nfs files are created when an open file is removed but is still being accessed
+.nfs*
+
+# Rust
+/target/
+
+### vscode ###
+.vscode/*
+!.vscode/settings.json
+!.vscode/tasks.json
+!.vscode/launch.json
+!.vscode/extensions.json
+*.code-workspace
+
+### Windows ###
+# Windows thumbnail cache files
+Thumbs.db
+Thumbs.db:encryptable
+ehthumbs.db
+ehthumbs_vista.db
+
+# Dump file
+*.stackdump
+
+# Folder config file
+[Dd]esktop.ini
+
+# Recycle Bin used on file shares
+$RECYCLE.BIN/
+
+# Windows shortcuts
+*.lnk
+
+# Conduit
 Rocket.toml
 conduit.toml
+conduit.db
+
+# Etc.
+**/*.rs.bk
@@ -0,0 +1,386 @@
+stages:
+  - build
+  - build docker image
+  - test
+  - upload artifacts
+  - publish
+
+variables:
+  GIT_SUBMODULE_STRATEGY: recursive
+  FF_USE_FASTZIP: 1
+  CACHE_COMPRESSION_LEVEL: fastest
+
+
+# --------------------------------------------------------------------- #
+#  Cargo: Compiling for different architectures                         #
+# --------------------------------------------------------------------- #
+
+.build-cargo-shared-settings:
+  stage: "build"
+  needs: []
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
+    - if: '$CI_COMMIT_TAG'
+  interruptible: true
+  image: "rust:latest"
+  tags: ["docker"]
+  cache:
+    paths:
+      - cargohome
+      - target/
+    key: "build_cache-$TARGET-release"
+  variables:
+    CARGO_PROFILE_RELEASE_LTO=true
+    CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1
+  before_script:
+    - 'echo "Building for target $TARGET"'
+    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
+    - 'apt-get update -yqq'
+    - 'echo "Installing packages: $NEEDED_PACKAGES"'
+    - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
+    - "rustup target add $TARGET"
+  script:
+    - time cargo build --target $TARGET --release
+    - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
+  artifacts:
+    expire_in: never
+
+
+build:release:cargo:x86_64-unknown-linux-gnu:
+  extends: .build-cargo-shared-settings
+  variables:
+    TARGET: "x86_64-unknown-linux-gnu"
+  artifacts:
+    name: "conduit-x86_64-unknown-linux-gnu"
+    paths:
+      - "conduit-x86_64-unknown-linux-gnu"
+    expose_as: "Conduit for x86_64-unknown-linux-gnu"
+
+build:release:cargo:armv7-unknown-linux-gnueabihf:
+  extends: .build-cargo-shared-settings
+  variables:
+    TARGET: "armv7-unknown-linux-gnueabihf"
+    NEEDED_PACKAGES: "build-essential gcc-arm-linux-gnueabihf g++-arm-linux-gnueabihf libc6-dev-armhf-cross"
+    CARGO_TARGET_ARMV7_UNKNOWN_LINUX_GNUEABIHF_LINKER: arm-linux-gnueabihf-gcc
+    CC_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-gcc
+    CXX_armv7_unknown_linux_gnueabihf: arm-linux-gnueabihf-g++
+  artifacts:
+    name: "conduit-armv7-unknown-linux-gnueabihf"
+    paths:
+      - "conduit-armv7-unknown-linux-gnueabihf"
+    expose_as: "Conduit for armv7-unknown-linux-gnueabihf"
+
+build:release:cargo:aarch64-unknown-linux-gnu:
+  extends: .build-cargo-shared-settings
+  variables:
+    TARGET: "aarch64-unknown-linux-gnu"
+    NEEDED_PACKAGES: "build-essential gcc-8-aarch64-linux-gnu g++-aarch64-linux-gnu libc6-dev-arm64-cross"
+    CARGO_TARGET_AARCH64_UNKNOWN_LINUX_GNU_LINKER: aarch64-linux-gnu-gcc
+    CC_aarch64_unknown_linux_gnu: aarch64-linux-gnu-gcc
+    CXX_aarch64_unknown_linux_gnu: aarch64-linux-gnu-g++
+    TARGET_CC: "/usr/bin/aarch64-linux-gnu-gcc-8"
+    TARGET_AR: "/usr/bin/aarch64-linux-gnu-gcc-ar-8"
+  artifacts:
+    name: "conduit-aarch64-unknown-linux-gnu"
+    paths:
+      - "conduit-aarch64-unknown-linux-gnu"
+    expose_as: "Conduit for aarch64-unknown-linux-gnu"
+
+build:release:cargo:x86_64-unknown-linux-musl:
+  extends: .build-cargo-shared-settings
+  image: "rust:alpine"
+  variables:
+    TARGET: "x86_64-unknown-linux-musl"
+  before_script:
+    - 'echo "Building for target $TARGET"'
+    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
+    - "rustup target add $TARGET"
+    - "apk add libc-dev"
+  artifacts:
+    name: "conduit-x86_64-unknown-linux-musl"
+    paths:
+      - "conduit-x86_64-unknown-linux-musl"
+    expose_as: "Conduit for x86_64-unknown-linux-musl"
+
+
+
+.cargo-debug-shared-settings:
+  extends: ".build-cargo-shared-settings"
+  rules:
+    - if: '$CI_COMMIT_BRANCH'
+    - if: '$CI_COMMIT_TAG'
+  cache:
+    key: "build_cache-$TARGET-debug"
+  script:
+    - "time cargo build --target $TARGET"
+    - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
+  artifacts:
+    expire_in: 4 weeks
+
+build:debug:cargo:x86_64-unknown-linux-gnu:
+  extends: ".cargo-debug-shared-settings"
+  variables:
+    TARGET: "x86_64-unknown-linux-gnu"
+  artifacts:
+    name: "conduit-debug-x86_64-unknown-linux-gnu"
+    paths:
+      - "conduit-debug-x86_64-unknown-linux-gnu"
+    expose_as: "Conduit DEBUG for x86_64-unknown-linux-gnu"
+
+build:debug:cargo:x86_64-unknown-linux-musl:
+  extends: ".cargo-debug-shared-settings"
+  image: "rust:alpine"
+  variables:
+    TARGET: "x86_64-unknown-linux-musl"
+  before_script:
+    - 'echo "Building for target $TARGET"'
+    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
+    - "rustup target add $TARGET"
+    - "apk add libc-dev"
+  artifacts:
+    name: "conduit-debug-x86_64-unknown-linux-musl"
+    paths:
+      - "conduit-debug-x86_64-unknown-linux-musl"
+    expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"
+
+
+
+# --------------------------------------------------------------------- #
+#  Cargo: Compiling deb packages for different architectures            #
+# --------------------------------------------------------------------- #
+
+
+.build-cargo-deb-shared-settings:
+  stage: "build"
+  needs: [ ]
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
+    - if: '$CI_COMMIT_TAG'
+  interruptible: true
+  image: "rust:latest"
+  tags: ["docker"]
+  cache:
+    paths:
+      - cargohome
+      - target/
+    key: "build_cache-deb-$TARGET"
+  before_script:
+    - 'echo "Building debian package for target $TARGET"'
+    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
+    - 'apt-get update -yqq'
+    - 'echo "Installing packages: $NEEDED_PACKAGES"'
+    - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
+    - "rustup target add $TARGET"
+    - "cargo install cargo-deb"
+  script:
+    - time cargo deb --target $TARGET
+    - 'mv target/$TARGET/debian/*.deb "conduit-$TARGET.deb"'
+
+build:cargo-deb:x86_64-unknown-linux-gnu:
+  extends: .build-cargo-deb-shared-settings
+  variables:
+    TARGET: "x86_64-unknown-linux-gnu"
+    NEEDED_PACKAGES: ""
+  artifacts:
+    name: "conduit-x86_64-unknown-linux-gnu.deb"
+    paths:
+      - "conduit-x86_64-unknown-linux-gnu.deb"
+    expose_as: "Debian Package x86_64"
+
+
+# --------------------------------------------------------------------- #
+#  Create and publish docker image                                      #
+# --------------------------------------------------------------------- #
+
+# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
+.docker-shared-settings:
+  stage: "build docker image"
+  needs: []
+  interruptible: true
+  image:
+    name: "gcr.io/kaniko-project/executor:debug"
+    entrypoint: [""]
+  tags: ["docker"]
+  variables:
+    # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache
+    KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache"
+  before_script:
+    - "mkdir -p /kaniko/.docker"
+    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'
+
+
+build:docker:next:
+  extends: .docker-shared-settings
+  needs:
+    - "build:release:cargo:x86_64-unknown-linux-musl"
+  script:
+    - >
+      /kaniko/executor
+      $KANIKO_CACHE_ARGS
+      --context $CI_PROJECT_DIR
+      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
+      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
+      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
+      --destination "$CI_REGISTRY_IMAGE/conduit:next"
+      --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine"
+      --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "next"'
+
+
+build:docker:master:
+  extends: .docker-shared-settings
+  needs:
+    - "build:release:cargo:x86_64-unknown-linux-musl"
+  script:
+    - >
+      /kaniko/executor
+      $KANIKO_CACHE_ARGS
+      --context $CI_PROJECT_DIR
+      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
+      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
+      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
+      --destination "$CI_REGISTRY_IMAGE/conduit:latest"
+      --destination "$CI_REGISTRY_IMAGE/conduit:latest-alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest-alpine"
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+
+build:docker:tags:
+  extends: .docker-shared-settings
+  needs:
+    - "build:release:cargo:x86_64-unknown-linux-musl"
+  script:
+    - >
+      /kaniko/executor
+      $KANIKO_CACHE_ARGS
+      --context $CI_PROJECT_DIR
+      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
+      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
+      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
+      --destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG"
+      --destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG-alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG-alpine"
+  rules:
+    - if: '$CI_COMMIT_TAG'
+
+
+
+# --------------------------------------------------------------------- #
+#  Run tests                                                            #
+# --------------------------------------------------------------------- #
+
+test:cargo:
+  stage: "test"
+  needs: [ ]
+  image: "rust:latest"
+  tags: [ "docker" ]
+  variables:
+    CARGO_HOME: "cargohome"
+  cache:
+    paths:
+      - target
+      - cargohome
+    key: test_cache
+  interruptible: true
+  before_script:
+    - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
+    - apt-get update -yqq
+    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
+    - rustup component add clippy rustfmt
+  script:
+    - rustc --version && cargo --version  # Print version info for debugging
+    - cargo fmt --all -- --check
+    - cargo test --workspace --verbose --locked
+    - cargo clippy
+
+test:sytest:
+  stage: "test"
+  allow_failure: true
+  needs:
+    - "build:debug:cargo:x86_64-unknown-linux-musl"
+  image:
+    name: "valkum/sytest-conduit:latest"
+    entrypoint: [ "" ]
+  tags: [ "docker" ]
+  variables:
+    PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
+  before_script:
+    - "mkdir -p /app"
+    - "cp ./conduit-debug-x86_64-unknown-linux-musl /app/conduit"
+    - "chmod +x /app/conduit"
+    - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
+    - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
+    - "cd /"
+  script:
+    - "SYTEST_EXIT_CODE=0"
+    - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
+    - "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap"
+    - "exit $SYTEST_EXIT_CODE"
+  artifacts:
+    when: always
+    paths:
+      - "$CI_PROJECT_DIR/sytest.xml"
+      - "$CI_PROJECT_DIR/results.tap"
+    reports:
+      junit: "$CI_PROJECT_DIR/sytest.xml"
+
+
+# --------------------------------------------------------------------- #
+#  Store binaries as package so they have download urls                 #
+# --------------------------------------------------------------------- #
+
+publish:package:
+  stage: "upload artifacts"
+  needs:
+    - "build:release:cargo:x86_64-unknown-linux-gnu"
+    - "build:release:cargo:armv7-unknown-linux-gnueabihf"
+    - "build:release:cargo:aarch64-unknown-linux-gnu"
+    - "build:release:cargo:x86_64-unknown-linux-musl"
+    - "build:cargo-deb:x86_64-unknown-linux-gnu"
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
+    - if: '$CI_COMMIT_TAG'
+  image: curlimages/curl:latest
+  tags: ["docker"]
+  variables:
+    GIT_STRATEGY: "none" # Don't need a clean copy of the code, we just operate on artifacts
+  script:
+    - 'BASE_URL="${CI_API_V4_URL}/projects/${CI_PROJECT_ID}/packages/generic/conduit-${CI_COMMIT_REF_SLUG}/build-${CI_PIPELINE_ID}"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu "${BASE_URL}/conduit-x86_64-unknown-linux-gnu"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-armv7-unknown-linux-gnueabihf "${BASE_URL}/conduit-armv7-unknown-linux-gnueabihf"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-aarch64-unknown-linux-gnu "${BASE_URL}/conduit-aarch64-unknown-linux-gnu"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-musl "${BASE_URL}/conduit-x86_64-unknown-linux-musl"'
+    - 'curl --header "JOB-TOKEN: $CI_JOB_TOKEN" --upload-file conduit-x86_64-unknown-linux-gnu.deb "${BASE_URL}/conduit-x86_64-unknown-linux-gnu.deb"'
+
+
+pages:
+  stage: "publish"
+  image:
+    name: "squidfunk/mkdocs-material"
+    entrypoint: [""]
+  tags: ["docker"]
+  needs: []
+  rules:
+    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+  script:
+    - "mkdocs build --site-dir public"
+  artifacts:
+    paths:
+      - "public"
@@ -0,0 +1,19 @@
+<!--
+If you're requesting a new feature, that isn't part of this project yet,
+then please consider filling out a "Feature Request" instead!
+
+If you need a hand setting up your conduit server, feel free to ask for help in the
+Conduit Matrix Chat: https://matrix.to/#/#conduit:fachschaften.org.
+-->
+
+### Description
+<!-- What did you do and what happened? Why is that bad? -->
+
+### System Configuration
+<!-- Other data that might help us debug this issue, like os, conduit version, database backend -->
+
+Conduit Version:
+Database backend (default is sqlite): sqlite
+
+
+/label ~conduit
@@ -0,0 +1,17 @@
+<!--
+If you want to report a bug or an error,
+then please consider filling out a "Bug Report" instead!
+-->
+
+
+### Is your feature request related to a problem? Please describe.
+
+<!-- Eg. I'm always frustrated when [...] -->
+
+
+### Describe the solution you'd like
+
+
+
+
+/label ~conduit
@@ -1,15 +0,0 @@
-# Headline
-
-### Description
-
-
-
-
-
-
-
-
-
-
-
-/label ~conduit
@@ -0,0 +1,8 @@
+
+<!-- Please describe your changes here -->
+
+-----------------------------------------------------------------------------
+
+- [ ] I ran `cargo fmt` and `cargo test`
+- [ ] I agree to release my code and all other changes of this MR under the Apache-2.0 license
+
@@ -0,0 +1,3 @@
+{
+    "rust-analyzer.procMacro.enable": true
+}
@@ -1,13 +0,0 @@
-Install docker:
-
-$ sudo apt install docker
-$ sudo usermod -aG docker $USER
-
-Then log out and back in.
-
-$ sudo systemctl start docker
-
-$ cargo install cross
-$ cross build --release --features tls_vendored --target armv7-unknown-linux-musleabihf
-
-The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit
@@ -6,7 +6,7 @@ authors = ["timokoesters <timo@koesters.xyz>"]
 homepage = "https://conduit.rs"
 repository = "https://gitlab.com/famedly/conduit"
 readme = "README.md"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2018"

 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -14,45 +14,45 @@ edition = "2018"
 [dependencies]
 # Used to handle requests
 # TODO: This can become optional as soon as proper configs are supported
-rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "93e62c86eddf7cc9a7fc40b044182f83f0d7d92a", features = ["tls"] } # Used to handle requests
-#rocket = { git = "https://github.com/timokoesters/Rocket.git", branch = "empty_parameters", default-features = false, features = ["tls"] }
+# rocket = { git = "https://github.com/SergioBenitez/Rocket.git", rev = "801e04bd5369eb39e126c75f6d11e1e9597304d8", features = ["tls"] } # Used to handle requests
+rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests

 # Used for matrix spec type definitions and helpers
-#ruma = { git = "https://github.com/ruma/ruma", features = ["rand", "appservice-api", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks", "unstable-exhaustive-types"], rev = "0a10afe6dacc2b7a50a8002c953d10b7fb4e37bc" }
-# ruma = { git = "https://github.com/DevinR528/ruma", features = ["rand", "client-api", "federation-api", "unstable-exhaustive-types", "unstable-pre-spec", "unstable-synapse-quirks"], branch = "verified-export" }
-ruma = { path = "../ruma/ruma", features = ["unstable-exhaustive-types", "rand", "client-api", "federation-api", "unstable-pre-spec", "unstable-synapse-quirks"] }
-
-# Used when doing state resolution
-# state-res = { git = "https://github.com/timokoesters/state-res", branch = "timo-spec-comp", features = ["unstable-pre-spec"] }
-# TODO: remove the gen-eventid feature
-#state-res = { git = "https://github.com/ruma/state-res", branch = "main", features = ["unstable-pre-spec", "gen-eventid"] }
-# state-res = { git = "https://github.com/ruma/state-res", rev = "791c66d73cf064d09db0cdf767d5fef43a343425", features = ["unstable-pre-spec", "gen-eventid"] }
-state-res = { path = "../state-res", features = ["unstable-pre-spec", "gen-eventid"] }
+#ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+ruma = { git = "https://github.com/ruma/ruma", rev = "a6a1224652912a957b09f136ec5da2686be6e0e2", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+#ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+#ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }

 # Used for long polling and federation sender, should be the same as rocket::tokio
-tokio = "1.2.0"
+tokio = "1.11.0"
 # Used for storing data permanently
-sled = { version = "0.34.6", default-features = false }
-# Used for emitting log entries
-log = "0.4.14"
+sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true }
+#sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] }
+
+# Used for the http request / response body type for Ruma endpoints used with reqwest
+bytes = "1.1.0"
 # Used for rocket<->ruma conversions
-http = "0.2.3"
+http = "0.2.4"
 # Used to find data directory for default db path
-directories = "3.0.1"
+directories = "3.0.2"
 # Used for ruma wrapper
-serde_json = { version = "1.0.64", features = ["raw_value"] }
+serde_json = { version = "1.0.67", features = ["raw_value"] }
 # Used for appservice registration files
-serde_yaml = "0.8.17"
+serde_yaml = "0.8.20"
 # Used for pdu definition
-serde = "1.0.123"
+serde = "1.0.130"
 # Used for secure identifiers
-rand = "0.8.3"
+rand = "0.8.4"
 # Used to hash passwords
 rust-argon2 = "0.8.3"
 # Used to send requests
-reqwest = { version = "0.11.1", features = ["rustls-tls"] }
+reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tls-native-roots", "socks"] }
+# Custom TLS verifier
+rustls = { version = "0.19.1", features = ["dangerous_configuration"] }
+rustls-native-certs = "0.5.0"
+webpki = "0.22.0"
 # Used for conduit::Error type
-thiserror = "1.0.24"
+thiserror = "1.0.28"
 # Used to generate thumbnails for images
 image = { version = "0.23.14", default-features = false, features = ["jpeg", "png", "gif"] }
 # Used to encode server public key
@@ -60,23 +60,33 @@ base64 = "0.13.0"
 # Used when hashing the state
 ring = "0.16.20"
 # Used when querying the SRV record of other servers
-trust-dns-resolver = "0.20.0"
+trust-dns-resolver = "0.20.3"
 # Used to find matching events for appservices
-regex = "1.4.3"
+regex = "1.5.4"
 # jwt jsonwebtokens
 jsonwebtoken = "7.2.0"
 # Performance measurements
-tracing = "0.1.25"
-opentelemetry = "0.12.0"
-tracing-subscriber = "0.2.16"
-tracing-opentelemetry = "0.11.0"
-opentelemetry-jaeger = "0.11.0"
-pretty_env_logger = "0.4.0"
+tracing = { version = "0.1.26", features = ["release_max_level_warn"] }
+tracing-subscriber = "0.2.20"
+tracing-flame = "0.1.0"
+opentelemetry = { version = "0.16.0", features = ["rt-tokio"] }
+opentelemetry-jaeger = { version = "0.15.0", features = ["rt-tokio"] }
+lru-cache = "0.1.2"
+rusqlite = { version = "0.25.3", optional = true, features = ["bundled"] }
+parking_lot = { version = "0.11.2", optional = true }
+crossbeam = { version = "0.8.1", optional = true }
+num_cpus = "1.13.0"
+threadpool = "1.8.1"
+heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true }
+thread_local = "1.1.3"

 [features]
-default = ["conduit_bin"]
+default = ["conduit_bin", "backend_sqlite"]
+backend_sled = ["sled"]
+backend_sqlite = ["sqlite"]
+backend_heed = ["heed", "crossbeam"]
+sqlite = ["rusqlite", "parking_lot", "crossbeam", "tokio/signal"]
 conduit_bin = [] # TODO: add rocket to this when it is optional
-tls_vendored = ["reqwest/native-tls-vendored"]

 [[bin]]
 name = "conduit"
@@ -99,13 +109,30 @@ instead of a server that has high scalability."""
 section = "net"
 priority = "optional"
 assets = [
-  ["debian/env.local", "etc/matrix-conduit/local", "644"],
  ["debian/README.Debian", "usr/share/doc/matrix-conduit/", "644"],
  ["README.md", "usr/share/doc/matrix-conduit/", "644"],
  ["target/release/conduit", "usr/sbin/matrix-conduit", "755"],
 ]
 conf-files = [
-  "/etc/matrix-conduit/local"
+  "/etc/matrix-conduit/conduit.toml"
 ]
 maintainer-scripts = "debian/"
 systemd-units = { unit-name = "matrix-conduit" }
+
+[profile.dev]
+lto = 'thin'
+incremental = true
+
+[profile.release]
+lto = 'thin'
+incremental = true
+
+codegen-units=32
+# If you want to make flamegraphs, enable debug info:
+# debug = true
+
+# For releases also try to max optimizations for dependencies:
+[profile.release.build-override]
+opt-level = 3
+[profile.release.package."*"]
+opt-level = 3
@@ -2,22 +2,51 @@

 ## Getting help

-If you run into any problems while setting up Conduit, write an email to `support@conduit.rs`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).
+If you run into any problems while setting up Conduit, write an email to `timo@koesters.xyz`, ask us in `#conduit:matrix.org` or [open an issue on GitLab](https://gitlab.com/famedly/conduit/-/issues/new).

 ## Installing Conduit

-You have to download the binary that fits your machine. Run `uname -m` to see
-what you need. Now copy the right url:
- x84_64: `https://conduit.rs/master/x86_64/conduit-bin`
- armv7: `https://conduit.rs/master/armv7/conduit-bin`
- armv8: `https://conduit.rs/master/armv8/conduit-bin`
- arm: `https://conduit.rs/master/arm/conduit-bin`
+You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:
+
+| CPU Architecture     | GNU (Ubuntu, Debian, ArchLinux, ...)  | MUSL (Alpine, ... )     |
+| -------------------- | ------------------------------------- | ----------------------- |
+| x84_64 / amd64       | [Download][x84_64-gnu]                | [Download][x84_64-musl] |
+| armv7 (Raspberry Pi) | [Download][armv7-gnu]                 | -                       |
+| armv8 / aarch64      | [Download][armv8-gnu]                 | -                       |
+
+[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:release:cargo:x86_64-unknown-linux-gnu
+
+[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
+
+[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:release:cargo:armv7-unknown-linux-gnueabihf
+
+[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:release:cargo:aarch64-unknown-linux-gnu

 ```bash
 $ sudo wget -O /usr/local/bin/matrix-conduit <url>
 $ sudo chmod +x /usr/local/bin/matrix-conduit
 ```

+Alternatively, you may compile the binary yourself using
+
+```bash
+$ cargo build --release
+```
+Note that this currently requires Rust 1.50.
+
+If you want to cross compile Conduit to another architecture, read the [Cross-Compile Guide](CROSS_COMPILE.md).
+
+
+## Adding a Conduit user
+
+While Conduit can run as any user it is usually better to use dedicated users for different services.
+This also allows you to make sure that the file permissions are correctly set up.
+
+In Debian you can use this command to create a Conduit user:
+
+```bash
+sudo adduser --system conduit --no-create-home
+```

 ## Setting up a systemd service

@@ -33,8 +62,8 @@ After=network.target

 [Service]
 Environment="CONDUIT_CONFIG=/etc/matrix-conduit/conduit.toml"
-User=root
-Group=root
+User=conduit
+Group=nogroup
 Restart=always
 ExecStart=/usr/local/bin/matrix-conduit

@@ -43,6 +72,7 @@ WantedBy=multi-user.target
 ```

 Finally, run
+
 ```bash
 $ sudo systemctl daemon-reload
 ```
@@ -51,6 +81,7 @@ $ sudo systemctl daemon-reload
 ## Creating the Conduit configuration file

 Now we need to create the Conduit's config file in `/etc/matrix-conduit/conduit.toml`. Paste this in **and take a moment to read it. You need to change at least the server name.**
+
 ```toml
 [global]
 # The server_name is the name of this server. It is used as a suffix for user
@@ -76,19 +107,39 @@ port = 6167
 # Max size for uploads
 max_request_size = 20_000_000 # in bytes

-# Disabling registration means no new users will be able to register on this server
-allow_registration = false
+# Enables registration. If set to false, no users can register on this server.
+allow_registration = true

 # Disable encryption, so no new encrypted rooms can be created
 # Note: existing rooms will continue to work
 allow_encryption = true
 allow_federation = true

-#cache_capacity = 1073741824 # in bytes, 1024 * 1024 * 1024
-#max_concurrent_requests = 4 # How many requests Conduit sends to other servers at the same time
+trusted_servers = ["matrix.org"]
+
+#max_concurrent_requests = 100 # How many requests Conduit sends to other servers at the same time
 #workers = 4 # default: cpu core count * 2

 address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy
+
+# The total amount of memory that the database will use.
+#db_cache_capacity_mb = 200
+```
+
+## Setting the correct file permissions
+
+As we are using a Conduit specific user we need to allow it to read the config.
+To do that you can run this command on Debian:
+
+```bash
+sudo chown -R conduit:nogroup /etc/matrix-conduit
+```
+
+If you use the default database path you also need to run this:
+
+```bash
+sudo mkdir -p /var/lib/matrix-conduit/conduit_db
+sudo chown -R conduit:nogroup /var/lib/matrix-conduit/conduit_db
 ```


@@ -99,7 +150,8 @@ This depends on whether you use Apache, Nginx or another web server.
 ### Apache

 Create `/etc/apache2/sites-enabled/050-conduit.conf` and copy-and-paste this:
-```
+
+```apache
 Listen 8448

 <VirtualHost *:443 *:8448>
@@ -107,16 +159,14 @@ Listen 8448
 ServerName your.server.name # EDIT THIS

 AllowEncodedSlashes NoDecode
-ProxyPass /_matrix/ http://localhost:6167/
-ProxyPassReverse /_matrix/ http://localhost:6167/
+ProxyPass /_matrix/ http://127.0.0.1:6167/_matrix/ nocanon
+ProxyPassReverse /_matrix/ http://127.0.0.1:6167/_matrix/

-Include /etc/letsencrypt/options-ssl-apache.conf
-SSLCertificateFile /etc/letsencrypt/live/your.server.name/fullchain.pem # EDIT THIS
-SSLCertificateKeyFile /etc/letsencrypt/live/your.server.name/privkey.pem # EDIT THIS
 </VirtualHost>
 ```

 **You need to make some edits again.** When you are done, run
+
 ```bash
 $ sudo systemctl reload apache2
 ```
@@ -126,18 +176,30 @@ $ sudo systemctl reload apache2

 If you use Nginx and not Apache, add the following server section inside the
 http section of `/etc/nginx/nginx.conf`
-```
+
+```nginx
 server {
-    listen 443;
-    listen 8448;
+    listen 443 ssl http2;
+    listen [::]:443 ssl http2;
+    listen 8448 ssl http2;
+    listen [::]:8448 ssl http2;
    server_name your.server.name; # EDIT THIS
+    merge_slashes off;

    location /_matrix/ {
-        proxy_pass http://localhost:6167/_matrix/;
+        proxy_pass http://127.0.0.1:6167$request_uri;
+        proxy_set_header Host $http_host;
+        proxy_buffering off;
    }
+
+    ssl_certificate /etc/letsencrypt/live/your.server.name/fullchain.pem; # EDIT THIS
+    ssl_certificate_key /etc/letsencrypt/live/your.server.name/privkey.pem; # EDIT THIS
+    ssl_trusted_certificate /etc/letsencrypt/live/your.server.name/chain.pem; # EDIT THIS
+    include /etc/letsencrypt/options-ssl-nginx.conf;
 }
 ```
 **You need to make some edits again.** When you are done, run
+
 ```bash
 $ sudo systemctl reload nginx
 ```
@@ -146,6 +208,7 @@ $ sudo systemctl reload nginx
 ## SSL Certificate

 The easiest way to get an SSL certificate, if you don't have one already, is to install `certbot` and run this:
+
 ```bash
 $ sudo certbot -d your.server.name
 ```
@@ -154,11 +217,26 @@ $ sudo certbot -d your.server.name
 ## You're done!

 Now you can start Conduit with:
+
 ```bash
 $ sudo systemctl start conduit
 ```

 Set it to start automatically when your system boots with:
+
 ```bash
 $ sudo systemctl enable conduit
 ```
+
+## How do I know it works?
+
+You can open <https://app.element.io>, enter your homeserver and try to register.
+
+You can also use these commands as a quick health check.
+
+```bash
+$ curl https://your.server.name/_matrix/client/versions
+$ curl https://your.server.name:8448/_matrix/client/versions
+```
+
+If you want to set up an appservice, take a look at the [Appservice Guide](APPSERVICES.md).
@@ -5,7 +5,12 @@

 ##########################  BUILD IMAGE  ##########################
 # Alpine build image to build Conduit's statically compiled binary
-FROM alpine:3.12 as builder
+FROM alpine:3.14 as builder
+
+# Install packages needed for building all crates
+RUN apk add --no-cache \
+        cargo \
+        openssl-dev

 # Specifies if the local project is build or if Conduit gets build
 # from the official git repository. Defaults to the git repo.
@@ -13,34 +18,30 @@ ARG LOCAL=false
 # Specifies which revision/commit is build. Defaults to HEAD
 ARG GIT_REF=origin/master

-# Add 'edge'-repository to get Rust 1.45
-RUN sed -i \
-	-e 's|v3\.12|edge|' \
-	/etc/apk/repositories
-
-# Install packages needed for building all crates
-RUN apk add --no-cache \
-        cargo \
-        openssl-dev
-
-
 # Copy project files from current folder
 COPY . .
 # Build it from the copied local files or from the official git repository
 RUN if [[ $LOCAL == "true" ]]; then \
+        mv ./docker/healthcheck.sh . ; \
+        echo "Building from local source..." ; \
        cargo install --path . ; \
    else \
-        cargo install --git "https://github.com/timokoesters/conduit.git" --rev ${GIT_REF}; \
+        echo "Building revision '${GIT_REF}' from online source..." ; \
+        cargo install --git "https://gitlab.com/famedly/conduit.git" --rev ${GIT_REF} ; \
+        echo "Loadings healthcheck script from online source..." ; \
+        wget "https://gitlab.com/famedly/conduit/-/raw/${GIT_REF#origin/}/docker/healthcheck.sh" ; \
    fi

 ########################## RUNTIME IMAGE ##########################
 # Create new stage with a minimal image for the actual
 # runtime image/container
-FROM alpine:3.12
+FROM alpine:3.14

 ARG CREATED
 ARG VERSION
-ARG GIT_REF=HEAD
+ARG GIT_REF=origin/master
+
+ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"

 # Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md
 # including a custom label specifying the build command
@@ -52,21 +53,22 @@ LABEL org.opencontainers.image.created=${CREATED} \
      org.opencontainers.image.description="A Matrix homeserver written in Rust" \
      org.opencontainers.image.url="https://conduit.rs/" \
      org.opencontainers.image.revision=${GIT_REF} \
-      org.opencontainers.image.source="https://git.koesters.xyz/timo/conduit.git" \
+      org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
      org.opencontainers.image.licenses="Apache-2.0" \
      org.opencontainers.image.documentation="" \
      org.opencontainers.image.ref.name="" \
      org.label-schema.docker.build="docker build . -t matrixconduit/matrix-conduit:latest --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ') --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)" \
      maintainer="Weasy666"

-# Standard port on which Rocket launches
-EXPOSE 8000
+# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose.
+EXPOSE 6167

 # Copy config files from context and the binary from
 # the "builder" stage to the current stage into folder
 # /srv/conduit and create data folder for database
 RUN mkdir -p /srv/conduit/.local/share/conduit
 COPY --from=builder /root/.cargo/bin/conduit /srv/conduit/
+COPY --from=builder ./healthcheck.sh /srv/conduit/

 # Add www-data user and group with UID 82, as used by alpine
 # https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
@@ -84,11 +86,8 @@ RUN apk add --no-cache \
        curl \
        libgcc

-# Create a volume for the database, to persist its contents
-VOLUME ["/srv/conduit/.local/share/conduit"]
-
 # Test if Conduit is still alive, uses the same endpoint as Element
-HEALTHCHECK --start-period=2s CMD curl --fail -s http://localhost:8000/_matrix/client/versions || curl -k --fail -s https://localhost:8000/_matrix/client/versions || exit 1
+HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh

 # Set user to www-data
 USER www-data
@@ -1,66 +1,73 @@
 # Conduit
 ### A Matrix homeserver written in Rust

-[![Liberapay](https://img.shields.io/liberapay/receives/timokoesters?logo=liberapay)](https://liberapay.com/timokoesters)
-[![Matrix](https://img.shields.io/matrix/conduit:conduit.rs?server_fqdn=conduit.koesters.xyz)](https://matrix.to/#/#conduit:matrix.org)
-
 #### What is the goal?

-A fast Matrix homeserver that's easy to set up and just works. You can install
+An efficient Matrix homeserver that's easy to set up and just works. You can install
 it on a mini-computer like the Raspberry Pi to host Matrix for your family,
 friends or company.

+
 #### Can I try it out?

-Yes! Just open a Matrix client (<https://app.element.io> or Element Android for
-example) and register on the `https://conduit.koesters.xyz` homeserver.
+Yes! You can test our Conduit instance by opening a Matrix client (<https://app.element.io> or Element Android for
+example) and registering on the `conduit.rs` homeserver.
+
+It is hosted on a ODROID HC 2 with 2GB RAM and a SAMSUNG Exynos 5422 CPU, which
+was used in the Samsung Galaxy S5. It joined many big rooms including Matrix
+HQ.
+
+
+#### What is the current status?
+
+As of 2021-09-01, Conduit is Beta, meaning you can join and participate in most
+Matrix rooms, but not all features are supported and you might run into bugs
+from time to time.
+
+There are still a few important features missing:
+
+- E2EE verification over federation
+- Outgoing read receipts, typing, presence over federation
+
+Check out the [Conduit 1.0 Release Milestone](https://gitlab.com/famedly/conduit/-/milestones/3).
+

 #### How can I deploy my own?

-##### Deploy
+Simple install (this was tested the most): [DEPLOY.md](DEPLOY.md)\
+Debian package: [debian/README.Debian](debian/README.Debian)\
+Docker: [docker/README.md](docker/README.md)

-Download or compile a conduit binary and call it from somewhere like a systemd script. [Read
-more](DEPLOY.md)
+If you want to connect an Appservice to Conduit, take a look at [APPSERVICES.md](APPSERVICES.md).

-##### Deploy using Docker
-
-Pull and run the docker image with
-
-``` bash
-docker pull matrixconduit/matrix-conduit:latest
-docker run -d -p 8448:8000 -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
-```
-
-Or build and run it with docker or docker-compose. [Read more](docker/README.md)
-
-#### What is it build on?
-
- [Ruma](https://www.ruma.io): Useful structures for endpoint requests and
-  responses that can be (de)serialized
- [Sled](https://github.com/spacejam/sled): A simple (key, value) database with
-  good performance
- [Rocket](https://rocket.rs): A flexible web framework
-
-#### What are the biggest things still missing?
-
- Appservices (Bridges and Bots)
- Most federation features (invites, e2ee)
- Push notifications on mobile
- Notification settings
- Lots of testing
-
-Also check out the [milestones](https://git.koesters.xyz/timo/conduit/milestones).

 #### How can I contribute?

 1. Look for an issue you would like to work on and make sure it's not assigned
   to other users
 2. Ask someone to assign the issue to you (comment on the issue or chat in
-   #conduit:matrix.org)
-3. Fork the repo and work on the issue. #conduit:matrix.org is happy to help :)
-4. Submit a PR
+   #conduit:nordgedanken.dev)
+3. Fork the repo and work on the issue. #conduit:nordgedanken.dev is happy to help :)
+4. Submit a MR
+
+
+#### Thanks to
+
+Thanks to Famedly, Prototype Fund (DLR and German BMBF) and all other individuals for financially supporting this project.
+
+Thanks to the contributors to Conduit and all libraries we use, for example:
+
+- Ruma: A clean library for the Matrix Spec in Rust
+- Rocket: A flexible web framework
+

 #### Donate

 Liberapay: <https://liberapay.com/timokoesters/>\
 Bitcoin: `bc1qnnykf986tw49ur7wx9rpw2tevpsztvar5x8w4n`
+
+
+#### Logo
+
+Lightning Bolt Logo: https://github.com/mozilla/fxemoji/blob/gh-pages/svgs/nature/u26A1-bolt.svg \
+Logo License: https://github.com/mozilla/fxemoji/blob/gh-pages/LICENSE.md
@@ -0,0 +1,16 @@
+[book]
+title = "Conduit Docs"
+author = "The Conduit contributors"
+description = "Conduit is a simple, fast and reliable chat server for the Matrix protocol"
+language = "en"
+src = "docs"
+
+[rust]
+edition = "2018"
+
+[build]
+build-dir = "public"
+create-missing = true
+
+[output.html.search]
+limit-results = 15
@@ -11,8 +11,8 @@
 # YOU NEED TO EDIT THIS
 #server_name = "your.server.name"

-# This is the only directly where Conduit will save its data
-database_path = "/var/lib/conduit/conduit.db"
+# This is the only directory where Conduit will save its data
+database_path = "/var/lib/conduit/"

 # The port Conduit will be running on. You need to set up a reverse proxy in
 # your web server (e.g. apache or nginx), so all requests to /_matrix on port
@@ -22,8 +22,8 @@ port = 6167
 # Max size for uploads
 max_request_size = 20_000_000 # in bytes

-# Disable registration. No new users will be able to register on this server
-#allow_registration = false
+# Enables registration. If set to false, no users can register on this server.
+allow_registration = true

 # Disable encryption, so no new encrypted rooms can be created
 # Note: existing rooms will continue to work
@@ -33,8 +33,15 @@ max_request_size = 20_000_000 # in bytes
 # Enable jaeger to support monitoring and troubleshooting through jaeger
 #allow_jaeger = false

-#cache_capacity = 1073741824 # in bytes, 1024 * 1024 * 1024
-#max_concurrent_requests = 4 # How many requests Conduit sends to other servers at the same time
+trusted_servers = ["matrix.org"]
+
+#max_concurrent_requests = 100 # How many requests Conduit sends to other servers at the same time
+#log = "info,state_res=warn,rocket=off,_=off,sled=off"
 #workers = 4 # default: cpu core count * 2

 address = "127.0.0.1" # This makes sure Conduit can only be reached using the reverse proxy
+
+proxy = "none" # more examples can be found at src/database/proxy.rs:6
+
+# The total amount of memory that the database will use.
+#db_cache_capacity_mb = 200
@@ -4,25 +4,25 @@ Conduit for Debian
 Configuration
 -------------

-When installed, Debconf handles the configuration of the homeserver (host)name,
-the address and port it listens on. These configuration variables end up in
-/etc/matrix-conduit/debian.
+When installed, Debconf generates the configuration of the homeserver
+(host)name, the address and port it listens on. This configuration ends up in
+/etc/matrix-conduit/conduit.toml.

 You can tweak more detailed settings by uncommenting and setting the variables
-in /etc/matrix-conduit/local. This involves settings such as the maximum file
-size for download/upload, enabling federation, etc.
+in /etc/matrix-conduit/conduit.toml. This involves settings such as the maximum
+file size for download/upload, enabling federation, etc.

 Running
 -------

 The package uses the matrix-conduit.service systemd unit file to start and
-stop Conduit. It loads the configuration files mentioned above to set up the
+stop Conduit. It loads the configuration file mentioned above to set up the
 environment before running the server.

-This package assumes by default that Conduit is placed behind a reverse proxy
-such as Apache or nginx. This default deployment entails just listening on
-127.0.0.1 and the free port 14004 and is reachable via a client using the URL
-http://localhost:14004.
+This package assumes by default that Conduit will be placed behind a reverse
+proxy such as Apache or nginx. This default deployment entails just listening
+on 127.0.0.1 and the free port 6167 and is reachable via a client using the URL
+http://localhost:6167.

 At a later stage this packaging may support also setting up TLS and running
 stand-alone.  In this case, however, you need to set up some certificates and
@@ -1,33 +0,0 @@
-# Conduit homeserver local configuration
-#
-# Conduit is an application based on the Rocket web framework.
-# Configuration of Conduit happens via Debconf (see the resulting config in
-# `/etc/matrix-conduit/debian`) and optionally by uncommenting and tweaking the
-# variables in this file below.
-
-# The maximum size of a Matrix HTTP requests in bytes.
-#
-# This mostly affects the size of files that can be downloaded/uploaded.
-# It defaults to 20971520 (20MB).
-#ROCKET_MAX_REQUEST_SIZE=20971520
-
-# Whether user registration is allowed.
-#
-# User registration is not disabled by default.
-#ROCKET_REGISTRATION_DISABLED=false
-
-# Whether encryption is enabled.
-#
-# (End-to-end) encryption is not disabled by default.
-#ROCKET_ENCRYPTION_DISABLED=false
-
-# Whether federation with other Matrix servers is enabled.
-#
-# Federation is not enabled by default; it is still experimental.
-#ROCKET_FEDERATION_ENABLED=false
-
-# The log level of the homeserver.
-#
-# The log level is "critical" by default.
-# Allowed values are: "off", "normal", "debug", "critical"
-#ROCKET_LOG="critical"
@@ -34,10 +34,7 @@ SystemCallFilter=@system-service
 SystemCallErrorNumber=EPERM
 StateDirectory=matrix-conduit

-Environment="ROCKET_ENV=production"
-Environment="ROCKET_DATABASE_PATH=/var/lib/matrix-conduit"
-EnvironmentFile=/etc/matrix-conduit/debian
-EnvironmentFile=/etc/matrix-conduit/local
+Environment="CONDUIT_CONFIG=/etc/matrix-conduit/conduit.toml"

 ExecStart=/usr/sbin/matrix-conduit
 Restart=on-failure
@@ -4,8 +4,8 @@ set -e
 . /usr/share/debconf/confmodule

 CONDUIT_CONFIG_PATH=/etc/matrix-conduit
-CONDUIT_CONFIG_FILE="$CONDUIT_CONFIG_PATH/debian"
-CONDUIT_DATABASE_PATH=/var/lib/matrix-conduit
+CONDUIT_CONFIG_FILE="${CONDUIT_CONFIG_PATH}/conduit.toml"
+CONDUIT_DATABASE_PATH=/var/lib/matrix-conduit/conduit_db

 case "$1" in
  configure)
@@ -13,7 +13,7 @@ case "$1" in
    if ! getent passwd _matrix-conduit > /dev/null ; then
      echo 'Adding system user for the Conduit Matrix homeserver' 1>&2
      adduser --system --group --quiet \
-        --home $CONDUIT_DATABASE_PATH \
+        --home "$CONDUIT_DATABASE_PATH" \
        --disabled-login \
        --force-badname \
        _matrix-conduit
@@ -25,48 +25,62 @@ case "$1" in
      chown _matrix-conduit "$CONDUIT_DATABASE_PATH"
    fi

-    # Write the debconf values in the config.
-    db_get matrix-conduit/hostname
-    ROCKET_SERVER_NAME="$RET"
-    db_get matrix-conduit/address
-    ROCKET_ADDRESS="$RET"
-    db_get matrix-conduit/port
-    ROCKET_PORT="$RET"
-    cat >"$CONDUIT_CONFIG_FILE" << EOF
-# Conduit homeserver Debian configuration
-#
-# Conduit is an application based on the Rocket web framework.
-# Configuration of Conduit happens via Debconf (of which the resulting config
-# is in this file) and optionally by uncommenting and tweaking the variables in
-# /etc/matrix-conduit/local.
+    if [ ! -e "$CONDUIT_CONFIG_FILE" ]; then
+      # Write the debconf values in the config.
+      db_get matrix-conduit/hostname
+      CONDUIT_SERVER_NAME="$RET"
+      db_get matrix-conduit/address
+      CONDUIT_ADDRESS="$RET"
+      db_get matrix-conduit/port
+      CONDUIT_PORT="$RET"
+      mkdir -p "$CONDUIT_CONFIG_PATH"
+      cat > "$CONDUIT_CONFIG_FILE" << EOF
+[global]
+# The server_name is the name of this server. It is used as a suffix for user
+# and room ids. Examples: matrix.org, conduit.rs
+# The Conduit server needs to be reachable at https://your.server.name/ on port
+# 443 (client-server) and 8448 (federation) OR you can create /.well-known
+# files to redirect requests. See
+# https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client
+# and https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server
+# for more information.
+server_name = "${CONDUIT_SERVER_NAME}"

-# THIS FILE IS GENERATED BY DEBCONF AND WILL BE OVERRIDDEN!
-#
-# Please make changes by running:
-#
-# \$ dpkg-reconfigure matrix-conduit
-#
-# or by providing overriding changes in /etc/matrix-conduit/local.
+# This is the only directory where Conduit will save its data.
+database_path = "${CONDUIT_DATABASE_PATH}"

-# The server (host)name of the Matrix homeserver.
-#
-# This is the hostname the homeserver will be reachable at via a client.
-ROCKET_SERVER_NAME="$ROCKET_SERVER_NAME"
-
-# The address the Matrix homeserver listens on.
-#
+# The address Conduit will be listening on.
 # By default the server listens on address 0.0.0.0. Change this to 127.0.0.1 to
 # only listen on the localhost when using a reverse proxy.
-ROCKET_ADDRESS="$ROCKET_ADDRESS"
+address = "${CONDUIT_ADDRESS}"

-# The port of the Matrix homeserver.
-#
-# This port is could be any available port if accessed by a reverse proxy.
-# By default the server listens on port 8000.
-ROCKET_PORT="$ROCKET_PORT"
+# The port Conduit will be running on. You need to set up a reverse proxy in
+# your web server (e.g. apache or nginx), so all requests to /_matrix on port
+# 443 and 8448 will be forwarded to the Conduit instance running on this port.
+port = ${CONDUIT_PORT}

-# THIS FILE IS GENERATED BY DEBCONF AND WILL BE OVERRIDDEN!
+# Max size for uploads
+max_request_size = 20_000_000 # in bytes
+
+# Enables registration. If set to false, no users can register on this server.
+allow_registration = true
+
+# Disable encryption, so no new encrypted rooms can be created.
+# Note: Existing rooms will continue to work.
+#allow_encryption = false
+#allow_federation = false
+
+# Enable jaeger to support monitoring and troubleshooting through jaeger.
+#allow_jaeger = false
+
+#max_concurrent_requests = 100 # How many requests Conduit sends to other servers at the same time
+#log = "info,state_res=warn,rocket=off,_=off,sled=off"
+#workers = 4 # default: cpu core count * 2
+
+# The total amount of memory that the database will use.
+#db_cache_capacity_mb = 200
 EOF
+    fi
    ;;
 esac

@@ -1,11 +1,16 @@
 #!/bin/sh
 set -e

+. /usr/share/debconf/confmodule
+
 CONDUIT_CONFIG_PATH=/etc/matrix-conduit
 CONDUIT_DATABASE_PATH=/var/lib/matrix-conduit

 case $1 in
  purge)
+    # Remove debconf changes from the db
+    db_purge
+
    # Per https://www.debian.org/doc/debian-policy/ch-files.html#behavior
    # "configuration files must be preserved when the package is removed, and
    #  only deleted when the package is purged."
@@ -16,6 +16,6 @@ Description: The listen address of the Matrix homeserver

 Template: matrix-conduit/port
 Type: string
-Default: 14004
+Default: 6167
 Description: The port of the Matrix homeserver
 This port is most often just accessed by a reverse proxy.
@@ -3,7 +3,7 @@ version: '3'

 services:
    homeserver:
-        ### If you already built the Conduit image with 'docker build' or want to use the Docker Hub image,
+        ### If you already built the Conduit image with 'docker build' or want to use a registry image,
        ### then you are ready to go.
        image: matrixconduit/matrix-conduit:latest
        ### If you want to build a fresh image from the sources, then comment the image line and uncomment the
@@ -12,36 +12,41 @@ services:
        # build:
        #     context: .
        #     args:
-        #         CREATED:
-        #         VERSION:
+        #         CREATED: '2021-03-16T08:18:27Z'
+        #         VERSION: '0.1.0'
        #         LOCAL: 'false'
-        #         GIT_REF: HEAD
+        #         GIT_REF: origin/master
        restart: unless-stopped
        ports:
-            - 8448:8000
+            - 8448:6167
        volumes:
            - db:/srv/conduit/.local/share/conduit
-            ### Uncomment if you want to use Rocket.toml to configure Conduit
-            ### Note: Set env vars will override Rocket.toml values
-            # - ./Rocket.toml:/srv/conduit/Rocket.toml
+            ### Uncomment if you want to use conduit.toml to configure Conduit
+            ### Note: Set env vars will override conduit.toml values
+            # - ./conduit.toml:/srv/conduit/conduit.toml
        environment:
-            ROCKET_SERVER_NAME: localhost:8000  # replace with your own name
+            CONDUIT_SERVER_NAME: localhost:6167  # replace with your own name
+            CONDUIT_TRUSTED_SERVERS: '["matrix.org"]'
+            CONDUIT_ALLOW_REGISTRATION: 'true'
            ### Uncomment and change values as desired
-            # ROCKET_LOG: normal  # Available levels are: off, debug, normal, critical
-            # ROCKET_PORT: 8000
-            # ROCKET_REGISTRATION_DISABLED: 'true'
-            # ROCKET_ENCRYPTION_DISABLED: 'true'
-            # ROCKET_FEDERATION_ENABLED: 'true'
-            # ROCKET_DATABASE_PATH: /srv/conduit/.local/share/conduit
-            # ROCKET_WORKERS: 10
-            # ROCKET_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB
+            # CONDUIT_ADDRESS: 0.0.0.0
+            # CONDUIT_PORT: 6167
+            # CONDUIT_CONFIG: '/srv/conduit/conduit.toml'  # if you want to configure purely by env vars, set this to an empty string ''
+            # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging
+            # CONDUIT_LOG: info  # default is: "info,rocket=off,_=off,sled=off"
+            # CONDUIT_ALLOW_JAEGER: 'false'
+            # CONDUIT_ALLOW_ENCRYPTION: 'false'
+            # CONDUIT_ALLOW_FEDERATION: 'false'
+            # CONDUIT_DATABASE_PATH: /srv/conduit/.local/share/conduit
+            # CONDUIT_WORKERS: 10
+            # CONDUIT_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB

    ### Uncomment if you want to use your own Element-Web App.
    ### Note: You need to provide a config.json for Element and you also need a second
    ###       Domain or Subdomain for the communication between Element and Conduit
    ### Config-Docs: https://github.com/vector-im/element-web/blob/develop/docs/config.md
    # element-web:
-    #     image: vectorim/riot-web:latest
+    #     image: vectorim/element-web:latest
    #     restart: unless-stopped
    #     ports:
    #         - 8009:80
@@ -51,4 +56,4 @@ services:
    #         - homeserver

 volumes:
-    db:
+  db:
@@ -17,13 +17,13 @@ The Dockerfile includes a few build arguments that should be supplied when build
 ARG LOCAL=false
 ARG CREATED
 ARG VERSION
-ARG GIT_REF=HEAD
+ARG GIT_REF=origin/master
 ```

 - **CREATED:** Date and time as string (date-time as defined by RFC 3339). Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.created`. Supply by it like this `$(date -u +'%Y-%m-%dT%H:%M:%SZ')`
 - **VERSION:** The SemVer version of Conduit, which is in the image. Will be used to create the Open Container Initiative compliant label `org.opencontainers.image.version`. If you have a `Cargo.toml` in your build context, you can get it with `$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)`
 - **LOCAL:** *(Optional)* A boolean value, specifies if the local build context should be used, or if the official repository will be cloned. If not supplied with the build command, it will default to `false`.
- **GIT_REF:** *(Optional)* A git ref, like `HEAD` or a commit ID. The supplied ref will be used to create the Open Container Initiative compliant label `org.opencontainers.image.revision` and will be the ref that is cloned from the repository when not building from the local context. If not supplied with the build command, it will default to `HEAD`.
+- **GIT_REF:** *(Optional)* A git ref, like `HEAD` or a commit ID. The supplied ref will be used to create the Open Container Initiative compliant label `org.opencontainers.image.revision` and will be the ref that is cloned from the repository when not building from the local context. If not supplied with the build command, it will default to `origin/master`.

 To build the image you can use the following command

@@ -40,16 +40,27 @@ which also will tag the resulting image as `matrixconduit/matrix-conduit:latest`
 After building the image you can simply run it with

 ``` bash
-docker run -d -p 8448:8000 -v db:/srv/conduit/.local/share/conduit -e ROCKET_SERVER_NAME="localhost:8000" matrixconduit/matrix-conduit:latest
+docker run -d -p 8448:6167 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
 ```

-For detached mode, you also need to use the `-d` flag. You can pass in more env vars as are shown here, for an overview of possible values, you can take a look at the `docker-compose.yml` file.
+or you can skip the build step and pull the image from one of the following registries:
+
+| Registry        | Image                                                                                                               | Size                                                                                        |
+| --------------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
+| Docker Hub      | [matrixconduit/matrix-conduit:latest](https://hub.docker.com/r/matrixconduit/matrix-conduit)                        | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) |
+| GitLab Registry | [registry.gitlab.com/famedly/conduit/conduit:latest](https://gitlab.com/famedly/conduit/container_registry/2134341) | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) |
+
+The `-d` flag lets the container run in detached mode. You now need to supply a `conduit.toml` config file, an example can be found [here](../conduit-example.toml).
+You can pass in different env vars to change config values on the fly. You can even configure Conduit completely by using env vars, but for that you need
+to pass `-e CONDUIT_CONFIG=""` into your container. For an overview of possible values, please take a look at the `docker-compose.yml` file.
+
 If you just want to test Conduit for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it.


 ## Docker-compose

-If the docker command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, use the [`docker-compose.traefik.yml`](docker-compose.traefik.yml) including [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) or the normal [`docker-compose.yml`](../docker-compose.yml) for every other reverse proxy.
+If the docker command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, use the [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) for Traefik (don't forget to remove `.traefik` from the filenames) or the normal [`docker-compose.yml`](../docker-compose.yml) for every other reverse proxy. Additional info about deploying
+Conduit can be found [here](../DEPLOY.md).


 ### Build
@@ -65,8 +76,57 @@ This will also start the container right afterwards, so if want it to run in det

 ### Run

-If you already have built the image, you can just start the container and everything else in the compose file in detached mode with:
+If you already have built the image or want to use one from the registries, you can just start the container and everything else in the compose file in detached mode with:

 ``` bash
 docker-compose up -d
 ```
+
+> **Note:** Don't forget to modify and adjust the compose file to your needs.
+
+### Use Traefik as Proxy
+
+As a container user, you probably know about Traefik. It is a easy to use reverse proxy for making containerized app and services available through the web. With the
+two provided files, [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml), it is
+equally easy to deploy and use Conduit, with a little caveat. If you already took a look at the files, then you should have seen the `well-known` service, and that is
+the little caveat. Traefik is simply a proxy and loadbalancer and is not able to serve any kind of content, but for Conduit to federate, we need to either expose ports
+`443` and `8448` or serve two endpoints `.well-known/matrix/client` and `.well-known/matrix/server`.
+
+With the service `well-known` we use a single `nginx` container that will serve those two files.
+
+So...step by step:
+
+1. Copy [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) from the repository and remove `.traefik` from the filenames.
+2. Open both files and modify/adjust them to your needs. Meaning, change the `CONDUIT_SERVER_NAME` and the volume host mappings according to your needs.
+3. Create the `conduit.toml` config file, an example can be found [here](../conduit-example.toml), or set `CONDUIT_CONFIG=""` and configure Conduit per env vars.
+4. Uncomment the `element-web` service if you want to host your own Element Web Client and create a `element_config.json`.
+5. Create the files needed by the `well-known` service.
+    - `./nginx/matrix.conf` (relative to the compose file, you can change this, but then also need to change the volume mapping)
+      ```nginx
+      server {
+          server_name <SUBDOMAIN>.<DOMAIN>;
+          listen      80 default_server;
+
+          location /.well-known/matrix/ {
+              root /var/www;
+              default_type application/json;
+              add_header Access-Control-Allow-Origin *;
+          }
+      }
+      ```
+    - `./nginx/www/.well-known/matrix/client` (relative to the compose file, you can change this, but then also need to change the volume mapping)
+      ```json
+      {
+          "m.homeserver": {
+              "base_url": "https://<SUBDOMAIN>.<DOMAIN>"
+          }
+      }
+      ```
+    - `./nginx/www/.well-known/matrix/server` (relative to the compose file, you can change this, but then also need to change the volume mapping)
+      ```json
+      {
+          "m.server": "<SUBDOMAIN>.<DOMAIN>:443"
+      }
+      ```
+6. Run `docker-compose up -d`
+7. Connect to your homeserver with your preferred client and create a user. You should do this immediatly after starting Conduit, because the first created user is the admin.
@@ -0,0 +1,69 @@
+# ---------------------------------------------------------------------------------------------------------
+# This Dockerfile is intended to be built as part of Conduit's CI pipeline.
+# It does not build Conduit in Docker, but just copies the matching build artifact from the build job.
+# As a consequence, this is not a multiarch capable image. It always expects and packages a x86_64 binary.
+#
+# It is mostly based on the normal Conduit Dockerfile, but adjusted in a few places to maximise caching.
+# Credit's for the original Dockerfile: Weasy666.
+# ---------------------------------------------------------------------------------------------------------
+
+FROM alpine:3.14
+
+# Install packages needed to run Conduit
+RUN apk add --no-cache \
+        ca-certificates \
+        curl \
+        libgcc
+
+ARG CREATED
+ARG VERSION
+ARG GIT_REF
+
+ENV CONDUIT_CONFIG="/srv/conduit/conduit.toml"
+
+# Labels according to https://github.com/opencontainers/image-spec/blob/master/annotations.md
+# including a custom label specifying the build command
+LABEL org.opencontainers.image.created=${CREATED} \
+      org.opencontainers.image.authors="Conduit Contributors" \
+      org.opencontainers.image.title="Conduit" \
+      org.opencontainers.image.version=${VERSION} \
+      org.opencontainers.image.vendor="Conduit Contributors" \
+      org.opencontainers.image.description="A Matrix homeserver written in Rust" \
+      org.opencontainers.image.url="https://conduit.rs/" \
+      org.opencontainers.image.revision=${GIT_REF} \
+      org.opencontainers.image.source="https://gitlab.com/famedly/conduit.git" \
+      org.opencontainers.image.licenses="Apache-2.0" \
+      org.opencontainers.image.documentation="" \
+      org.opencontainers.image.ref.name=""
+
+# Standard port on which Conduit launches. You still need to map the port when using the docker command or docker-compose.
+EXPOSE 6167
+
+# create data folder for database
+RUN mkdir -p /srv/conduit/.local/share/conduit
+
+# Copy the Conduit binary into the image at the latest possible moment to maximise caching:
+COPY ./conduit-x86_64-unknown-linux-musl /srv/conduit/conduit
+COPY ./docker/healthcheck.sh /srv/conduit/
+
+# Add www-data user and group with UID 82, as used by alpine
+# https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
+RUN set -x ; \
+    addgroup -Sg 82 www-data 2>/dev/null ; \
+    adduser -S -D -H -h /srv/conduit -G www-data -g www-data www-data 2>/dev/null ; \
+    addgroup www-data www-data 2>/dev/null && exit 0 ; exit 1
+
+# Change ownership of Conduit files to www-data user and group
+RUN chown -cR www-data:www-data /srv/conduit
+RUN chmod +x /srv/conduit/healthcheck.sh
+
+
+# Test if Conduit is still alive, uses the same endpoint as Element
+HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh
+
+# Set user to www-data
+USER www-data
+# Set container home directory
+WORKDIR /srv/conduit
+# Run Conduit
+ENTRYPOINT [ "/srv/conduit/conduit" ]
@@ -10,6 +10,29 @@ services:
            - "traefik.http.routers.to-conduit.rule=Host(`<SUBDOMAIN>.<DOMAIN>`)"  # Change to the address on which Conduit is hosted
            - "traefik.http.routers.to-conduit.tls=true"
            - "traefik.http.routers.to-conduit.tls.certresolver=letsencrypt"
+            - "traefik.http.routers.to-conduit.middlewares=cors-headers@docker"
+
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*"
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"
+
+    # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container
+    # to serve those two as static files. If you want to use a different way, delete or comment the below service, here
+    # and in the docker-compose file.
+    well-known:
+        labels:
+            - "traefik.enable=true"
+            - "traefik.docker.network=proxy"
+
+            - "traefik.http.routers.to-matrix-wellknown.rule=Host(`<SUBDOMAIN>.<DOMAIN>`) && PathPrefix(`/.well-known/matrix`)"
+            - "traefik.http.routers.to-matrix-wellknown.tls=true"
+            - "traefik.http.routers.to-matrix-wellknown.tls.certresolver=letsencrypt"
+            - "traefik.http.routers.to-matrix-wellknown.middlewares=cors-headers@docker"
+
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOriginList=*"
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"
+

    ### Uncomment this if you uncommented Element-Web App in the docker-compose.yml
    # element-web:
@@ -12,35 +12,51 @@ services:
        # build:
        #     context: .
        #     args:
-        #         CREATED:
-        #         VERSION:
+        #         CREATED: '2021-03-16T08:18:27Z'
+        #         VERSION: '0.1.0'
        #         LOCAL: 'false'
-        #         GIT_REF: HEAD
+        #         GIT_REF: origin/master
        restart: unless-stopped
        volumes:
            - db:/srv/conduit/.local/share/conduit
-            ### Uncomment if you want to use Rocket.toml to configure Conduit
-            ### Note: Set env vars will override Rocket.toml values
-            # - ./Rocket.toml:/srv/conduit/Rocket.toml
+            ### Uncomment if you want to use conduit.toml to configure Conduit
+            ### Note: Set env vars will override conduit.toml values
+            # - ./conduit.toml:/srv/conduit/conduit.toml
        networks:
            - proxy
        environment:
-            ROCKET_SERVER_NAME: localhost:8000  # replace with your own name
+            CONDUIT_SERVER_NAME: localhost:6167  # replace with your own name
+            CONDUIT_TRUSTED_SERVERS: '["matrix.org"]'
+            CONDUIT_ALLOW_REGISTRATION : 'true'
            ### Uncomment and change values as desired
-            # ROCKET_LOG: normal  # Available levels are: off, debug, normal, critical
-            # ROCKET_PORT: 8000
-            # ROCKET_REGISTRATION_DISABLED: 'true'
-            # ROCKET_ENCRYPTION_DISABLED: 'true'
-            # ROCKET_DATABASE_PATH: /srv/conduit/.local/share/conduit
-            # ROCKET_WORKERS: 10
-            # ROCKET_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB
+            # CONDUIT_ADDRESS: 0.0.0.0
+            # CONDUIT_PORT: 6167
+            # CONDUIT_CONFIG: '/srv/conduit/conduit.toml'  # if you want to configure purely by env vars, set this to an empty string ''
+            # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging
+            # CONDUIT_LOG: info  # default is: "info,rocket=off,_=off,sled=off"
+            # CONDUIT_ALLOW_JAEGER: 'false'
+            # CONDUIT_ALLOW_ENCRYPTION: 'false'
+            # CONDUIT_ALLOW_FEDERATION: 'false'
+            # CONDUIT_DATABASE_PATH: /srv/conduit/.local/share/conduit
+            # CONDUIT_WORKERS: 10
+            # CONDUIT_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB
+
+    # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container
+    # to serve those two as static files. If you want to use a different way, delete or comment the below service, here
+    # and in the docker-compose override file.
+    well-known:
+        image: nginx:latest
+        restart: unless-stopped
+        volumes:
+            - ./nginx/matrix.conf:/etc/nginx/conf.d/matrix.conf # the config to serve the .well-known/matrix files
+            - ./nginx/www:/var/www/ # location of the client and server .well-known-files

    ### Uncomment if you want to use your own Element-Web App.
    ### Note: You need to provide a config.json for Element and you also need a second
    ###       Domain or Subdomain for the communication between Element and Conduit
    ### Config-Docs: https://github.com/vector-im/element-web/blob/develop/docs/config.md
    # element-web:
-    #     image: vectorim/riot-web:latest
+    #     image: vectorim/element-web:latest
    #     restart: unless-stopped
    #     volumes:
    #         - ./element_config.json:/app/config.json
@@ -53,7 +69,7 @@ volumes:
    db:

 networks:
-    # This is the network Traefik listens to, if you network has a different
+    # This is the network Traefik listens to, if your network has a different
    # name, don't forget to change it here and in the docker-compose.override.yml
    proxy:
        external: true
@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# If the port is not specified as env var, take it from the config file
+if [ -z ${CONDUIT_PORT} ]; then
+  CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')
+fi
+
+# The actual health check.
+# We try to first get a response on HTTP and when that fails on HTTPS and when that fails, we exit with code 1.
+# TODO: Change this to a single curl call. Do we have a config value that we can check for that?
+curl --fail -s "http://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
+    curl -k --fail -s "https://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
+    exit 1
@@ -0,0 +1,16 @@
+# Summary
+
+[Home](index.md)
+
+- [Installation](installation.md)
+    - [Prerequisites](installation/prerequisites.md)
+    - [From Binaries](installation/manual.md)
+    - [From Packages](installation/packages.md)
+    - [With Docker](installation/docker.md)
+- [Configuration](configuration.md)
+    - [Config options](configuration/configuration.md)
+    - [Appservices](configuration/appservices.md)
+- [Contribute](contribute.md)
+    - [Basics](development/basics.md)
+    - [Cross compilation](development/cross-compilation.md)
+    - [Tests & CI](development/tests-ci.md)
@@ -0,0 +1 @@
+# Configuration
@@ -0,0 +1,2 @@
+# Configuring Conduit
+
@@ -0,0 +1,220 @@
+# Configuring Conduit
+
+Conduit can be configured via a config file (conventionally called Conduit.toml) or environment variables. If a config
+file exists and environment variables are set, environment variables overwrite config options.
+
+You absolutely need to set the environment variable `CONDUIT_CONFIG_FILE` to either point to a config file (
+e.g. `CONDUIT_CONFIG_FILE=/etc/conduit/Conduit.toml`) or to an empty string (`CONDUIT_CONFIG_FILE=''`) if you want to
+configure Conduit with just environment variables.
+
+## Mandatory config options
+
+Mandatory variables must be configured in order for Conduit to run properly.
+
+### Server Name
+
+- Config file key: `server_name`
+- Envirnoment variable: `CONDUIT_SERVER_NAME`
+- Default value: _None, you will need to choose your own._
+
+The server_name is the name of this server. It is used as a suffix for user and room ids. Example: If you set it
+to `conduit.rs`, your usernames will look like `@somebody:conduit.rs`.
+
+The Conduit server needs to be reachable at https://your.server.name/ on port 443 (client-server) and 8448 (
+server-server) OR you can create /.well-known files to redirect requests. See
+the [Client-Server specs](https://matrix.org/docs/spec/client_server/latest#get-well-known-matrix-client) and
+the [Server-Server specs](https://matrix.org/docs/spec/server_server/r0.1.4#get-well-known-matrix-server) for more
+information.
+
+### Database Path
+
+- Config file key: `database_path`
+- Envirnoment variable: `CONDUIT_DATABASE_PATH`
+- Default value: _None, but many people like to use `/var/lib/conduit/`_.
+
+A **directory** where Conduit stores its database and media files. This directory must exist, have enough free space and
+be readable and writable by the user Conduit is running as.
+
+What does _enough free space_ mean? It heavily on the amount of messages your Conduit server will see and the amount and
+size of media files users on your Conduit server send. As a rule of thumb, you should have at least 10 GB of free space
+left. You should be comfortable for quite some time with 50 GB.
+
+### TCP Port
+
+- Config file key: `port`
+- Environment variable: `CONDUIT_PORT`
+- Default value: _None, but many people like to use `6167`_.
+
+The TCP port Conduit will listen on for connections. The port needs to be free (no other program is listeing on it).
+
+Conduit does currently (2021-09) not offer HTTPS by itself. Only unencrypted HTTP requests will be accepted on this
+port. Unless you know what you are doing, this port should not be exposed to the internet. Instead, use a reverse proxy
+capable of doing TLS to offer your Conduit server to the internet via HTTPS. See [TODO] for example configurations.
+
+## Optional configuration options
+
+These config options come with defaults and don't need to be configured for Conduit to run. That said, you should still
+check them to make sure that your Conduit server behaves like you want it to do.
+
+### Maximum request size
+
+- Config file key: `max_request_size`
+- Environment variable: `CONDUIT_MAX_REQUEST_SIZE`
+- Default value: `20_000_000` (~= 20 MB)
+
+The maximum size in bytes for incoming requests to Conduit. You can use underscores to improve readability.
+
+This will effectively limit the size for images, videos and other files users on your Conduit server can send.
+
+### Allow Registration?
+
+- Config file key: `allow_registration`
+- Environment variable: `CONDUIT_ALLOW_REGISTRATION`
+- Default value: `true`
+- Possible values: `true`, `false`
+
+It this is set to `false`, no new users can register accounts on your Conduit server. Already registered users will not
+be affected from this setting and can continue to user your server.
+
+The first user to ever register on your Conduit server will be considered the admin account and is automatically invited
+into the admin room.
+
+### Allow Encryption?
+
+- Config file key: `allow_encryption`
+- Environment variable: `CONDUIT_ALLOW_ENCRYPTION`
+- Default value: `true`
+- Possible values: `true`, `false`
+
+If this is set to `false`, Conduit disables the ability for users to create encrypted chats. Existing encrypted chats
+may continue to work.
+
+### Allow federation?
+
+- Config file key: `allow_federation`
+- Environment variable: `CONDUIT_ALLOW_FEDERATION`
+- Default value: `false`
+- Possible values: `true`, `false`
+
+Federation means that users from different Matrix servers can chat with each other. E.g. `@mathew:matrix.org` can chat
+with `@timo:conduit.rs`.
+
+If this option is set to `false`, users on your Conduit server can only talk with other users on your Conduit server.
+
+Federation with other servers needs to happen over HTTPS, so make sure you have set up a reverse proxy.
+
+### Jaeger Tracing
+
+- Config file key: `allow_jaeger`
+- Environment variable: `CONDUIT_ALLOW_JAEGER`
+- Default value: `false`
+- Possible values: `true`, `false`
+
+Enable Jaeger to support monitoring and troubleshooting through Jaeger.
+
+If you don't know what Jaeger is, you can safely leave this set to `false`.
+
+### Trusted servers
+
+- Config file key: `trusted_servers`
+- Environment variable: `CONDUIT_TRUSTED_SERVERS`
+- Default value: `[]`
+- Possible values: JSON-Array of server domains, e.g. `["matrix.org"]` or `["matrix.org", "conduit.rs"]`.
+
+Matrix servers have so-called "server keys", which authenticate messages from their users. Because your Conduit server
+might not know the server keys from every server it encounters, it can ask a _trusted server_ for them. This speeds
+things up for rooms with people from a lot of different servers.
+
+You should only set this to include trustworthy servers. Most people consider a good default to be `["matrix.org"]`.
+
+Only relevant if you have federation enabled.
+
+### Limit amount of concurrent requests
+
+- Config file key: `max_concurrent_requests`
+- Environment variable: `CONDUIT_MAX_CONCURRENT_REQUESTS`
+- Default value: `100`
+- Suggested values: `1` - `1000` (u16)
+
+How many requests Conduit can make at the same time. This affects federation with other Matrix servers, push
+notifications and app_services.
+
+// TODO Timo: When does it make sense to change this?
+
+### Configure logging
+
+- Config file key: `log`
+- Environment variable: `CONDUIT_LOG`
+- Default value: `info,state_res=warn,rocket=off,_=off,sled=off`
+
+Configures which kind of messages Conduit logs.
+
+> // TODO: Better and more thorough explanation
+
+### Worker threads
+
+- Config file key: `workers`
+- Environment variable: `CONDUIT_WORKERS`
+- Default value: cpu core count * 2
+- Possible values: // TODO
+
+> // TODO: Which thing exactly threads? What not?
+
+### Listening address
+
+- Config file key: `address`
+- Environment variable: `CONDUIT_ADDRESS`
+- Default value: `127.0.0.1`
+- Possible values: Valid IP addresses.
+
+Which IP address conduit is listening on. 127.0.0.1 means that Conduit can only be accessed from the same server or
+through a reverse proxy on that server. If you want it to be accessible from any network interface (which you should
+not, because other matrix servers should talk to your Conduit via a reverse proxy and not directly), you can set it
+to `0.0.0.0`.
+
+### Database cache capacity
+
+- Config file key: `db_cache_capacity_mb`
+- Environment variable: `CONDUIT_DB_CACHE_CAPACITY_MB`
+- Default value: `200`
+- Possible values: `true`, `false`
+
+The total amount of memory (RAM) that the database cache will be able to use.
+
+> // TODO: this needs clearification: In RAM or on disk and for what exactly?
+
+### PDU cache capacity
+
+- Config file key: `pdu_cache_capacity`
+- Environment variable: `CONDUIT_PDU_CACHE_CAPACITY`
+- Default value: `100_000`
+- Suggested values: `1_000` - `1_000_000` (u32)
+
+The total capacity (read: number of items) the pdu cache can hold in memory. Setting this to a lower number may slow
+Conduit down, as it must fetch more data from the database. Increasing it will mean that Conduit will start to use more
+memory as the cache slowly fills up.
+
+### SQLite WAL clean interval
+
+- Config file key: `sqlite_wal_clean_second_interval`
+- Environment variable: `CONDUIT_SQLITE_WAL_CLEAN_SECOND_INTERVAL`
+- Default value: `60` (every 60 seconds)
+- Suggested values: `1` - `3600` (u32)
+
+How often the WAL file should be cleaned up. The WAL file will be written to until cleaned up, after which it restarts
+writing from the beginning.
+
+The file's size will correspond to how long it could write to it in one go. (e.g. if conduit writes 100MB of data to the
+database inbetween that period, the file will grow to 100MB). You can read more about that in
+the [SQLite Docs](https://www.sqlite.org/draft/wal.html).
+
+Reducing this down too much can offset the benefits of using a WAL at all. However, having this too high can result in a
+large WAL file.
+
+Only relevant when using SQLite as the database.
+
+### Still undocumented config options
+
+- `tracing_flame`
+- `proxy`
+- `jwt_secret`
@@ -0,0 +1 @@
+# Contribute
@@ -0,0 +1 @@
+# Basics
@@ -0,0 +1,13 @@
+# Cross compilation
+
+Install docker:
+
+```bash
+sudo apt install docker
+sudo usermod -aG docker $USER
+exec sudo su -l $USER
+sudo systemctl start docker
+cargo install cross
+cross build --release --target armv7-unknown-linux-musleabihf
+```
+The cross-compiled binary is at target/armv7-unknown-linux-musleabihf/release/conduit
@@ -0,0 +1 @@
+# Tests & CI
@@ -0,0 +1,52 @@
+# Conduit
+
+Conduit is a simple, fast and reliable chat server for the [Matrix] protocol written in [Rust].
+
+-----
+> Note: This project is work-in-progress. Do *not* rely on it yet.
+
+## What is Matrix?
+
+[Matrix] is an open network for secure and decentralized
+communication. It allows you to chat with friends even if they are using
+another servers and client. You can even use bridges to communicate with users
+outside of Matrix, like a community on Discord or your family on Hangouts.
+
+## Why Conduit?
+
+Conduit is an open-source server implementation of the [Matrix
+Specification] with a focus on easy setup and low
+system requirements, making it very easy to set up.
+
+Other server implementations try to be extremely scalable, which makes sense if
+the goal is to support millions of users on a single instance, but makes
+smaller deployments a lot more inefficient. Conduit tries to keep it simple but
+takes full advantage of that, for example by using an in-memory database for
+[huge performance gains](https://github.com/timokoesters/romeo-and-juliet-benchmark).
+
+The future for Conduit in peer-to-peer Matrix (every client contains a server)
+is also bright.
+
+Conduit tries to be reliable by using the Rust programming language and paying
+close attention to error handling to make sure that evil clients, misbehaving
+servers or even a partially broken database will not cause the whole server to
+stop working.
+
+## Chat with us!
+
+We have a room on Matrix: [#conduit:matrix.org](https://matrix.to/#/#conduit:matrix.org)
+
+You can also contact us using:
+- Matrix: [@timo:koesters.xyz](https://matrix.to/#/@timo:koesters.xyz)
+- Email: [conduit@koesters.xyz](mailto:conduit@koesters.xyz)
+
+
+## Donate
+
+Liberapay: <https://liberapay.com/timokoesters/>\
+Bitcoin: `bc1qnnykf986tw49ur7wx9rpw2tevpsztvar5x8w4n`
+
+
+[Matrix]: https://matrix.org/
+[Rust]: https://rust-lang.org
+[Matrix Specification]: https://matrix.org/docs/spec
@@ -0,0 +1 @@
+# Installation
@@ -0,0 +1,2 @@
+:::warning Test
+:::
@@ -0,0 +1,12 @@
+# Distribution packages
+
+## Debian / Ubuntu
+
+[@paul:luon.net](https://matrix.to/#/@paul:luon.net) plans to package Conduit for Debian as soon as it reaches 1.0.
+Until it is available in the official repos, you can install the development version of it manually:
+
+```bash
+sudo apt-get install ca-certificates
+wget --https-only -O /tmp/conduit.deb https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu.deb?job=build:cargo-deb:x86_64-unknown-linux-gnu
+sudo dpkg -i /tmp/conduit.deb
+```
@@ -0,0 +1,31 @@
+# Prerequisites for running Conduit
+
+You'll need:
+
+- A domain. Commonly cost about $10/year.
+- A Linux server with a stable internet connection, at least 500 MB of RAM and some disk space for messages and
+  attachments. Commonly start at $5/month.
+- Some basic knowledge about using a shell, SSH and configuring and protecting a server.
+  
+  
+## A word of caution:
+
+Don't underestimate the toll of administrating your own server.
+Conduit can't protect your conversations if your server gets compromised or deleted.
+
+Make sure that you got:
+
+- Automatic security updates
+    - On Ubuntu/Debian: Set up unattended-upgrades
+    - On RHEL/CentOS: Have a look at yum-cron
+- A firewall blocking all but the needed incoming ports
+    - ufw is an easy interface for the linux firewall
+- Protection against automatic attacks
+    - fail2ban scans logs and bans IPs which try to brute force their way into your server.
+    - Disable ssh login for root and switch from password to key based authentication.
+- Automated backups
+    - Most VPS hosting companies offer whole server backups for a small fee.
+    - Or run your own backup with something like borg.
+- A way to get notified if your disk fills up.
+    - If you send too much cat videos to your friends, Conduit might at some point become unable to
+      store any important messages.
@@ -0,0 +1,40 @@
+# About Matrix Homeservers
+
+Matrix homeservers manage its users chats. Every Matrix username includes the homserver it belongs to:
+`@alice:matrix.org` means that the `matrix.org` homeserver hosts a user called `@alice`.
+Every time someone chats with Alice, the `matrix.org` homeserver stores these messages.
+When `@alice:matrix.org` talks with `@adelaide:matrix.org`, that's easy. Both users use the same server.
+
+But how can `@bob:vector.tld`, who uses the `vector.tld` homeserver, exchange messages with `@alice:matrix.org`?
+This is where it get's a bit more complicated.
+
+## Matrix Homeserver discovery
+
+The Matrix specification specifies multiple ways how servers can discover and then talk to each other.
+Let's look at the most common one:
+
+### .well-known files
+
+At first, the only information a server has about a user (e.g. `@bob:vector.tld`) is its homeserver name: `vector.tld`.
+It then makes a HTTP GET request to `https://vector.tld/.well-known/matrix/server`.
+In the ideal case, this file contains a content like this: 
+
+```json
+{
+  "m.server": "matrix.vector.tld:443"
+}
+```
+
+This translates to: The matrix homeserver software for users with a username ending on `vector.tld`
+can be found at the address `matrix.vector.tld` at port 443 (which is the common port for HTTPS).
+
+The homeserver on it's quest to find `@bob:vector.tld` now contacts `matrix.vector.tld:443` and is then
+able to exchange chat messages with it.
+
+
+### Why so complicated?
+
+Organizations often don't want to run their Matrix server on the same machine that hosts their website,
+but `@foo:matrix.evil.corp` usernames are ugly and everyone wants to be `@foo:evil.corp`.
+
+To solve that problem, Matrix implements this extra step via a .well-known file or a DNS entry.
@@ -0,0 +1,20 @@
+site_name: Conduit Docs
+site_description: Conduit is a simple, fast and reliable chat server for the Matrix protocol
+theme:
+  name: material
+repo_url: https://gitlab.com/famedly/conduit
+nav:
+  - Home: index.md
+  - Installation:
+      - Prerequisites: installation/prerequisites.md
+      - Distribution Packages: installation/packages.md
+      - Manual: installation/manual.md
+      - Docker: installation/docker.md
+  - Configuration:
+      - Conduit.toml: configuration/conduit.toml.md
+      - Appservices: configuration/appservices.md
+  - Development:
+      - Basics: development/basics.md
+      - Cross compilation: development/cross-compilation.md
+      - Tests & CI: development/tests-ci.md
+
@@ -1 +1 @@
-1.47.0
+1.52
@@ -1 +1,2 @@
+unstable_features = true
 imports_granularity="Crate"
@@ -1,14 +1,15 @@
 use crate::{utils, Error, Result};
-use http::header::{HeaderValue, CONTENT_TYPE};
-use log::warn;
-use ruma::api::OutgoingRequest;
+use bytes::BytesMut;
+use ruma::api::{IncomingResponse, OutgoingRequest, SendAccessToken};
 use std::{
    convert::{TryFrom, TryInto},
    fmt::Debug,
+    mem,
    time::Duration,
 };
+use tracing::warn;

-pub async fn send_request<T: OutgoingRequest>(
+pub(crate) async fn send_request<T: OutgoingRequest>(
    globals: &crate::database::globals::Globals,
    registration: serde_yaml::Value,
    request: T,
@@ -20,8 +21,9 @@ where
    let hs_token = registration.get("hs_token").unwrap().as_str().unwrap();

    let mut http_request = request
-        .try_into_http_request(&destination, Some(""))
-        .unwrap();
+        .try_into_http_request::<BytesMut>(destination, SendAccessToken::IfRequired(""))
+        .unwrap()
+        .map(|body| body.freeze());

    let mut parts = http_request.uri().clone().into_parts();
    let old_path_and_query = parts.path_and_query.unwrap().as_str().to_owned();
@@ -38,67 +40,55 @@ where
    );
    *http_request.uri_mut() = parts.try_into().expect("our manipulation is always valid");

-    http_request.headers_mut().insert(
-        CONTENT_TYPE,
-        HeaderValue::from_str("application/json").unwrap(),
-    );
-
    let mut reqwest_request = reqwest::Request::try_from(http_request)
        .expect("all http requests are valid reqwest requests");

    *reqwest_request.timeout_mut() = Some(Duration::from_secs(30));

    let url = reqwest_request.url().clone();
-    let reqwest_response = globals.reqwest_client().execute(reqwest_request).await;
+    let mut response = globals
+        .reqwest_client()?
+        .build()?
+        .execute(reqwest_request)
+        .await?;

-    // Because reqwest::Response -> http::Response is complicated:
-    match reqwest_response {
-        Ok(mut reqwest_response) => {
-            let status = reqwest_response.status();
-            let mut http_response = http::Response::builder().status(status);
-            let headers = http_response.headers_mut().unwrap();
+    // reqwest::Response -> http::Response conversion
+    let status = response.status();
+    let mut http_response_builder = http::Response::builder()
+        .status(status)
+        .version(response.version());
+    mem::swap(
+        response.headers_mut(),
+        http_response_builder
+            .headers_mut()
+            .expect("http::response::Builder is usable"),
+    );

-            for (k, v) in reqwest_response.headers_mut().drain() {
-                if let Some(key) = k {
-                    headers.insert(key, v);
-                }
-            }
+    let body = response.bytes().await.unwrap_or_else(|e| {
+        warn!("server error: {}", e);
+        Vec::new().into()
+    }); // TODO: handle timeout

-            let status = reqwest_response.status();
-
-            let body = reqwest_response
-                .bytes()
-                .await
-                .unwrap_or_else(|e| {
-                    warn!("server error: {}", e);
-                    Vec::new().into()
-                }) // TODO: handle timeout
-                .into_iter()
-                .collect::<Vec<_>>();
-
-            if status != 200 {
-                warn!(
-                    "Appservice returned bad response {} {}\n{}\n{:?}",
-                    destination,
-                    status,
-                    url,
-                    utils::string_from_bytes(&body)
-                );
-            }
-
-            let response = T::IncomingResponse::try_from(
-                http_response
-                    .body(body)
-                    .expect("reqwest body is valid http body"),
-            );
-            response.map_err(|_| {
-                warn!(
-                    "Appservice returned invalid response bytes {}\n{}",
-                    destination, url
-                );
-                Error::BadServerResponse("Server returned bad response.")
-            })
-        }
-        Err(e) => Err(e.into()),
+    if status != 200 {
+        warn!(
+            "Appservice returned bad response {} {}\n{}\n{:?}",
+            destination,
+            status,
+            url,
+            utils::string_from_bytes(&body)
+        );
    }
+
+    let response = T::IncomingResponse::try_from_http_response(
+        http_response_builder
+            .body(body)
+            .expect("reqwest body is valid http body"),
+    );
+    response.map_err(|_| {
+        warn!(
+            "Appservice returned invalid response bytes {}\n{}",
+            destination, url
+        );
+        Error::BadServerResponse("Server returned bad response.")
+    })
 }
@@ -1,8 +1,11 @@
-use std::{collections::BTreeMap, convert::TryInto};
+use std::{
+    collections::BTreeMap,
+    convert::{TryFrom, TryInto},
+    sync::Arc,
+};

-use super::{State, DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH};
-use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma};
-use log::info;
+use super::{DEVICE_ID_LENGTH, SESSION_ID_LENGTH, TOKEN_LENGTH};
+use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma};
 use ruma::{
    api::client::{
        error::ErrorKind,
@@ -11,6 +14,7 @@ use ruma::{
                change_password, deactivate, get_username_availability, register, whoami,
                ThirdPartyIdRemovalStatus,
            },
+            contact::get_contacts,
            uiaa::{AuthFlow, UiaaInfo},
        },
    },
@@ -21,8 +25,10 @@ use ruma::{
        },
        EventType,
    },
-    RoomAliasId, RoomId, RoomVersionId, UserId,
+    identifiers::RoomName,
+    push, RoomAliasId, RoomId, RoomVersionId, UserId,
 };
+use tracing::info;

 use register::RegistrationKind;
 #[cfg(feature = "conduit_bin")]
@@ -34,15 +40,19 @@ const GUEST_NAME_LENGTH: usize = 10;
 ///
 /// Checks if a username is valid and available on this server.
 ///
-/// - Returns true if no user or appservice on this server claimed this username
-/// - This will not reserve the username, so the username might become invalid when trying to register
+/// Conditions for returning true:
+/// - The user id is not historical
+/// - The server name of the user id matches this server
+/// - No user or appservice on this server already claimed this username
+///
+/// Note: This will not reserve the username, so the username might become invalid when trying to register
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/register/available", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_register_available_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_username_availability::Request<'_>>,
 ) -> ConduitResult<get_username_availability::Response> {
    // Validate user id
@@ -74,21 +84,25 @@ pub async fn get_register_available_route(
 ///
 /// Register an account on this homeserver.
 ///
-/// - Returns the device id and access_token unless `inhibit_login` is true
-/// - When registering a guest account, all parameters except initial_device_display_name will be
-/// ignored
-/// - Creates a new account and a device for it
-/// - The account will be populated with default account data
+/// You can use [`GET /_matrix/client/r0/register/available`](fn.get_register_available_route.html)
+/// to check if the user id is valid and available.
+///
+/// - Only works if registration is enabled
+/// - If type is guest: ignores all parameters except initial_device_display_name
+/// - If sender is not appservice: Requires UIAA (but we only use a dummy stage)
+/// - If type is not guest and no username is given: Always fails after UIAA check
+/// - Creates a new account and populates it with default account data
+/// - If `inhibit_login` is false: Creates a device and returns device id and access_token
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/register", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn register_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<register::Request<'_>>,
 ) -> ConduitResult<register::Response> {
-    if !db.globals.allow_registration() {
+    if !db.globals.allow_registration() && !body.from_appservice {
        return Err(Error::BadRequest(
            ErrorKind::Forbidden,
            "Registration has been disabled.",
@@ -123,7 +137,7 @@ pub async fn register_route(
    ))?;

    // Check if username is creative enough
-    if !missing_username && db.users.exists(&user_id)? {
+    if db.users.exists(&user_id)? {
        return Err(Error::BadRequest(
            ErrorKind::UserInUse,
            "Desired user ID is already taken.",
@@ -143,17 +157,31 @@ pub async fn register_route(

    if !body.from_appservice {
        if let Some(auth) = &body.auth {
-            let (worked, uiaainfo) =
-                db.uiaa
-                    .try_auth(&user_id, "".into(), auth, &uiaainfo, &db.users, &db.globals)?;
+            let (worked, uiaainfo) = db.uiaa.try_auth(
+                &UserId::parse_with_server_name("", db.globals.server_name())
+                    .expect("we know this is valid"),
+                "".into(),
+                auth,
+                &uiaainfo,
+                &db.users,
+                &db.globals,
+            )?;
            if !worked {
                return Err(Error::Uiaa(uiaainfo));
            }
        // Success!
-        } else {
+        } else if let Some(json) = body.json_body {
            uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-            db.uiaa.create(&user_id, "".into(), &uiaainfo)?;
+            db.uiaa.create(
+                &UserId::parse_with_server_name("", db.globals.server_name())
+                    .expect("we know this is valid"),
+                "".into(),
+                &uiaainfo,
+                &json,
+            )?;
            return Err(Error::Uiaa(uiaainfo));
+        } else {
+            return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
        }
    }

@@ -167,26 +195,31 @@ pub async fn register_route(
    let password = if is_guest {
        None
    } else {
-        body.password.clone()
-    }
-    .unwrap_or_default();
+        body.password.as_deref()
+    };

    // Create user
-    db.users.create(&user_id, &password)?;
+    db.users.create(&user_id, password)?;

-    // Initial data
+    // Default to pretty displayname
+    let displayname = format!("{} ⚡️", user_id.localpart());
+    db.users
+        .set_displayname(&user_id, Some(displayname.clone()))?;
+
+    // Initial account data
    db.account_data.update(
        None,
        &user_id,
        EventType::PushRules,
        &ruma::events::push_rules::PushRulesEvent {
            content: ruma::events::push_rules::PushRulesEventContent {
-                global: crate::push_rules::default_pushrules(&user_id),
+                global: push::Ruleset::server_default(&user_id),
            },
        },
        &db.globals,
    )?;

+    // Inhibit login does not work for guests
    if !is_guest && body.inhibit_login {
        return Ok(register::Response {
            access_token: None,
@@ -207,7 +240,7 @@ pub async fn register_route(
    // Generate new token for the device
    let token = utils::random_string(TOKEN_LENGTH);

-    // Add device
+    // Create device for this account
    db.users.create_device(
        &user_id,
        &device_id,
@@ -215,16 +248,28 @@ pub async fn register_route(
        body.initial_device_display_name.clone(),
    )?;

-    // If this is the first user on this server, create the admins room
-    if db.users.count() == 1 {
+    // If this is the first user on this server, create the admin room
+    if db.users.count()? == 1 {
        // Create a user for the server
        let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name())
            .expect("@conduit:server_name is valid");

-        db.users.create(&conduit_user, "")?;
+        db.users.create(&conduit_user, None)?;

        let room_id = RoomId::new(db.globals.server_name());

+        db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
+
+        let mutex_state = Arc::clone(
+            db.globals
+                .roomid_mutex_state
+                .write()
+                .unwrap()
+                .entry(room_id.clone())
+                .or_default(),
+        );
+        let state_lock = mutex_state.lock().await;
+
        let mut content = ruma::events::room::create::CreateEventContent::new(conduit_user.clone());
        content.federate = true;
        content.predecessor = None;
@@ -242,6 +287,7 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        // 2. Make conduit bot join
@@ -254,6 +300,8 @@ pub async fn register_route(
                    avatar_url: None,
                    is_direct: None,
                    third_party_invite: None,
+                    blurhash: None,
+                    reason: None,
                })
                .expect("event is valid, we just created it"),
                unsigned: None,
@@ -263,6 +311,7 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        // 3. Power levels
@@ -275,18 +324,8 @@ pub async fn register_route(
                event_type: EventType::RoomPowerLevels,
                content: serde_json::to_value(
                    ruma::events::room::power_levels::PowerLevelsEventContent {
-                        ban: 50.into(),
-                        events: BTreeMap::new(),
-                        events_default: 0.into(),
-                        invite: 50.into(),
-                        kick: 50.into(),
-                        redact: 50.into(),
-                        state_default: 50.into(),
                        users,
-                        users_default: 0.into(),
-                        notifications: ruma::events::room::power_levels::NotificationPowerLevels {
-                            room: 50.into(),
-                        },
+                        ..Default::default()
                    },
                )
                .expect("event is valid, we just created it"),
@@ -297,6 +336,7 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        // 4.1 Join Rules
@@ -314,6 +354,7 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        // 4.2 History Visibility
@@ -333,6 +374,7 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        // 4.3 Guest Access
@@ -350,18 +392,18 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        // 6. Events implied by name and topic
+        let room_name =
+            Box::<RoomName>::try_from(format!("{} Admin Room", db.globals.server_name()))
+                .expect("Room name is valid");
        db.rooms.build_and_append_pdu(
            PduBuilder {
                event_type: EventType::RoomName,
-                content: serde_json::to_value(
-                    name::NameEventContent::new("Admin Room".to_owned()).map_err(|_| {
-                        Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.")
-                    })?,
-                )
-                .expect("event is valid, we just created it"),
+                content: serde_json::to_value(name::NameEventContent::new(Some(room_name)))
+                    .expect("event is valid, we just created it"),
                unsigned: None,
                state_key: Some("".to_owned()),
                redacts: None,
@@ -369,6 +411,7 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        db.rooms.build_and_append_pdu(
@@ -385,6 +428,7 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        // Room alias
@@ -407,6 +451,7 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;

        db.rooms.set_alias(&alias, Some(&room_id), &db.globals)?;
@@ -421,6 +466,8 @@ pub async fn register_route(
                    avatar_url: None,
                    is_direct: None,
                    third_party_invite: None,
+                    blurhash: None,
+                    reason: None,
                })
                .expect("event is valid, we just created it"),
                unsigned: None,
@@ -430,16 +477,19 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;
        db.rooms.build_and_append_pdu(
            PduBuilder {
                event_type: EventType::RoomMember,
                content: serde_json::to_value(member::MemberEventContent {
                    membership: member::MembershipState::Join,
-                    displayname: None,
+                    displayname: Some(displayname),
                    avatar_url: None,
                    is_direct: None,
                    third_party_invite: None,
+                    blurhash: None,
+                    reason: None,
                })
                .expect("event is valid, we just created it"),
                unsigned: None,
@@ -449,6 +499,7 @@ pub async fn register_route(
            &user_id,
            &room_id,
            &db,
+            &state_lock,
        )?;

        // Send welcome message
@@ -456,8 +507,8 @@ pub async fn register_route(
            PduBuilder {
                event_type: EventType::RoomMessage,
                content: serde_json::to_value(message::MessageEventContent::text_html(
-                        "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing `/join #conduit:matrix.org`. **Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.** Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
-                        "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing <code>/join #conduit:matrix.org</code>. <strong>Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.</strong> Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
+                        "## Thank you for trying out Conduit!\n\nConduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Website: https://conduit.rs\n> Git and Documentation: https://gitlab.com/famedly/conduit\n> Report issues: https://gitlab.com/famedly/conduit/-/issues\n\nHere are some rooms you can join (by typing the command):\n\nConduit room (Ask questions and get notified on updates):\n`/join #conduit:fachschaften.org`\n\nConduit lounge (Off-topic, only Conduit users are allowed to join)\n`/join #conduit-lounge:conduit.rs`".to_owned(),
+                        "<h2>Thank you for trying out Conduit!</h2>\n<p>Conduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.</p>\n<p>Helpful links:</p>\n<blockquote>\n<p>Website: https://conduit.rs<br>Git and Documentation: https://gitlab.com/famedly/conduit<br>Report issues: https://gitlab.com/famedly/conduit/-/issues</p>\n</blockquote>\n<p>Here are some rooms you can join (by typing the command):</p>\n<p>Conduit room (Ask questions and get notified on updates):<br><code>/join #conduit:fachschaften.org</code></p>\n<p>Conduit lounge (Off-topic, only Conduit users are allowed to join)<br><code>/join #conduit-lounge:conduit.rs</code></p>\n".to_owned(),
                ))
                .expect("event is valid, we just created it"),
                unsigned: None,
@@ -467,12 +518,13 @@ pub async fn register_route(
            &conduit_user,
            &room_id,
            &db,
+            &state_lock,
        )?;
    }

    info!("{} registered on this server", user_id);

-    db.flush().await?;
+    db.flush()?;

    Ok(register::Response {
        access_token: Some(token),
@@ -486,16 +538,23 @@ pub async fn register_route(
 ///
 /// Changes the password of this account.
 ///
-/// - Invalidates all other access tokens if logout_devices is true
-/// - Deletes all other devices and most of their data (to-device events, last seen, etc.) if
-/// logout_devices is true
+/// - Requires UIAA to verify user password
+/// - Changes the password of the sender user
+/// - The password hash is calculated using argon2 with 32 character salt, the plain password is
+/// not saved
+///
+/// If logout_devices is true it does the following for each device except the sender device:
+/// - Invalidates access token
+/// - Deletes device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets to-device events
+/// - Triggers device list updates
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/account/password", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn change_password_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<change_password::Request<'_>>,
 ) -> ConduitResult<change_password::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -513,7 +572,7 @@ pub async fn change_password_route(

    if let Some(auth) = &body.auth {
        let (worked, uiaainfo) = db.uiaa.try_auth(
-            &sender_user,
+            sender_user,
            sender_device,
            auth,
            &uiaainfo,
@@ -524,36 +583,40 @@ pub async fn change_password_route(
            return Err(Error::Uiaa(uiaainfo));
        }
    // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
        uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(sender_user, sender_device, &uiaainfo, &json)?;
        return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
    }

-    db.users.set_password(&sender_user, &body.new_password)?;
+    db.users
+        .set_password(sender_user, Some(&body.new_password))?;

    if body.logout_devices {
        // Logout all devices except the current one
        for id in db
            .users
-            .all_device_ids(&sender_user)
+            .all_device_ids(sender_user)
            .filter_map(|id| id.ok())
            .filter(|id| id != sender_device)
        {
-            db.users.remove_device(&sender_user, &id)?;
+            db.users.remove_device(sender_user, &id)?;
        }
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(change_password::Response.into())
+    Ok(change_password::Response {}.into())
 }

 /// # `GET _matrix/client/r0/account/whoami`
 ///
-/// Get user_id of this account.
+/// Get user_id of the sender user.
 ///
-/// - Also works for Application Services
+/// Note: Also works for Application Services
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/account/whoami", data = "<body>")
@@ -569,11 +632,13 @@ pub async fn whoami_route(body: Ruma<whoami::Request>) -> ConduitResult<whoami::

 /// # `POST /_matrix/client/r0/account/deactivate`
 ///
-/// Deactivate this user's account
+/// Deactivate sender user account.
 ///
 /// - Leaves all rooms and rejects all invitations
 /// - Invalidates all access tokens
-/// - Deletes all devices
+/// - Deletes all device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets all to-device events
+/// - Triggers device list updates
 /// - Removes ability to log in again
 #[cfg_attr(
    feature = "conduit_bin",
@@ -581,7 +646,7 @@ pub async fn whoami_route(body: Ruma<whoami::Request>) -> ConduitResult<whoami::
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn deactivate_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<deactivate::Request<'_>>,
 ) -> ConduitResult<deactivate::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -599,8 +664,8 @@ pub async fn deactivate_route(

    if let Some(auth) = &body.auth {
        let (worked, uiaainfo) = db.uiaa.try_auth(
-            &sender_user,
-            &sender_device,
+            sender_user,
+            sender_device,
            auth,
            &uiaainfo,
            &db.users,
@@ -610,18 +675,28 @@ pub async fn deactivate_route(
            return Err(Error::Uiaa(uiaainfo));
        }
    // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
        uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(sender_user, sender_device, &uiaainfo, &json)?;
        return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
    }

    // Leave all joined rooms and reject all invitations
-    for room_id in db
+    // TODO: work over federation invites
+    let all_rooms = db
        .rooms
-        .rooms_joined(&sender_user)
-        .chain(db.rooms.rooms_invited(&sender_user))
-    {
+        .rooms_joined(sender_user)
+        .chain(
+            db.rooms
+                .rooms_invited(sender_user)
+                .map(|t| t.map(|(r, _)| r)),
+        )
+        .collect::<Vec<_>>();
+
+    for room_id in all_rooms {
        let room_id = room_id?;
        let event = member::MemberEventContent {
            membership: member::MembershipState::Leave,
@@ -629,8 +704,20 @@ pub async fn deactivate_route(
            avatar_url: None,
            is_direct: None,
            third_party_invite: None,
+            blurhash: None,
+            reason: None,
        };

+        let mutex_state = Arc::clone(
+            db.globals
+                .roomid_mutex_state
+                .write()
+                .unwrap()
+                .entry(room_id.clone())
+                .or_default(),
+        );
+        let state_lock = mutex_state.lock().await;
+
        db.rooms.build_and_append_pdu(
            PduBuilder {
                event_type: EventType::RoomMember,
@@ -639,21 +726,39 @@ pub async fn deactivate_route(
                state_key: Some(sender_user.to_string()),
                redacts: None,
            },
-            &sender_user,
+            sender_user,
            &room_id,
            &db,
+            &state_lock,
        )?;
    }

    // Remove devices and mark account as deactivated
-    db.users.deactivate_account(&sender_user)?;
+    db.users.deactivate_account(sender_user)?;

    info!("{} deactivated their account", sender_user);

-    db.flush().await?;
+    db.flush()?;

    Ok(deactivate::Response {
        id_server_unbind_result: ThirdPartyIdRemovalStatus::NoSupport,
    }
    .into())
 }
+
+/// # `GET _matrix/client/r0/account/3pid`
+///
+/// Get a list of third party identifiers associated with this account.
+///
+/// - Currently always returns empty list
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/account/3pid", data = "<body>")
+)]
+pub async fn third_party_route(
+    body: Ruma<get_contacts::Request>,
+) -> ConduitResult<get_contacts::Response> {
+    let _sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    Ok(get_contacts::Response::new(Vec::new()).into())
+}
@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Ruma};
 use regex::Regex;
 use ruma::{
    api::{
@@ -16,15 +15,25 @@ use ruma::{
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, put};

+/// # `PUT /_matrix/client/r0/directory/room/{roomAlias}`
+///
+/// Creates a new room alias on this server.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/directory/room/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn create_alias_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<create_alias::Request<'_>>,
 ) -> ConduitResult<create_alias::Response> {
+    if body.room_alias.server_name() != db.globals.server_name() {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Alias is from another server.",
+        ));
+    }
+
    if db.rooms.id_from_alias(&body.room_alias)?.is_some() {
        return Err(Error::Conflict("Alias already exists."));
    }
@@ -32,40 +41,60 @@ pub async fn create_alias_route(
    db.rooms
        .set_alias(&body.room_alias, Some(&body.room_id), &db.globals)?;

-    db.flush().await?;
+    db.flush()?;

    Ok(create_alias::Response::new().into())
 }

+/// # `DELETE /_matrix/client/r0/directory/room/{roomAlias}`
+///
+/// Deletes a room alias from this server.
+///
+/// - TODO: additional access control checks
+/// - TODO: Update canonical alias event
 #[cfg_attr(
    feature = "conduit_bin",
    delete("/_matrix/client/r0/directory/room/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_alias_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_alias::Request<'_>>,
 ) -> ConduitResult<delete_alias::Response> {
+    if body.room_alias.server_name() != db.globals.server_name() {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Alias is from another server.",
+        ));
+    }
+
    db.rooms.set_alias(&body.room_alias, None, &db.globals)?;

-    db.flush().await?;
+    // TODO: update alt_aliases?
+
+    db.flush()?;

    Ok(delete_alias::Response::new().into())
 }

+/// # `GET /_matrix/client/r0/directory/room/{roomAlias}`
+///
+/// Resolve an alias locally or over federation.
+///
+/// - TODO: Suggest more servers to join via
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/directory/room/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_alias_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_alias::Request<'_>>,
 ) -> ConduitResult<get_alias::Response> {
    get_alias_helper(&db, &body.room_alias).await
 }

-pub async fn get_alias_helper(
+pub(crate) async fn get_alias_helper(
    db: &Database,
    room_alias: &RoomAliasId,
 ) -> ConduitResult<get_alias::Response> {
@@ -83,18 +112,24 @@ pub async fn get_alias_helper(
    }

    let mut room_id = None;
-    match db.rooms.id_from_alias(&room_alias)? {
+    match db.rooms.id_from_alias(room_alias)? {
        Some(r) => room_id = Some(r),
        None => {
-            for (_id, registration) in db.appservice.iter_all().filter_map(|r| r.ok()) {
+            for (_id, registration) in db.appservice.all()? {
                let aliases = registration
                    .get("namespaces")
                    .and_then(|ns| ns.get("aliases"))
-                    .and_then(|users| users.get("regex"))
-                    .and_then(|regex| regex.as_str())
-                    .and_then(|regex| Regex::new(regex).ok());
+                    .and_then(|aliases| aliases.as_sequence())
+                    .map_or_else(Vec::new, |aliases| {
+                        aliases
+                            .iter()
+                            .filter_map(|aliases| Regex::new(aliases.get("regex")?.as_str()?).ok())
+                            .collect::<Vec<_>>()
+                    });

-                if aliases.map_or(false, |aliases| aliases.is_match(room_alias.as_str()))
+                if aliases
+                    .iter()
+                    .any(|aliases| aliases.is_match(room_alias.as_str()))
                    && db
                        .sending
                        .send_appservice_request(
@@ -105,7 +140,7 @@ pub async fn get_alias_helper(
                        .await
                        .is_ok()
                {
-                    room_id = Some(db.rooms.id_from_alias(&room_alias)?.ok_or_else(|| {
+                    room_id = Some(db.rooms.id_from_alias(room_alias)?.ok_or_else(|| {
                        Error::bad_config("Appservice lied to us. Room does not exist.")
                    })?);
                    break;
@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::api::client::{
    error::ErrorKind,
    r0::backup::{
@@ -13,57 +12,66 @@ use ruma::api::client::{
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, post, put};

+/// # `POST /_matrix/client/r0/room_keys/version`
+///
+/// Creates a new backup.
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/unstable/room_keys/version", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn create_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<create_backup::Request>,
 ) -> ConduitResult<create_backup::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
    let version = db
        .key_backups
-        .create_backup(&sender_user, &body.algorithm, &db.globals)?;
+        .create_backup(sender_user, &body.algorithm, &db.globals)?;

-    db.flush().await?;
+    db.flush()?;

    Ok(create_backup::Response { version }.into())
 }

+/// # `PUT /_matrix/client/r0/room_keys/version/{version}`
+///
+/// Update information about an existing backup. Only `auth_data` can be modified.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn update_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<update_backup::Request<'_>>,
 ) -> ConduitResult<update_backup::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
    db.key_backups
-        .update_backup(&sender_user, &body.version, &body.algorithm, &db.globals)?;
+        .update_backup(sender_user, &body.version, &body.algorithm, &db.globals)?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(update_backup::Response.into())
+    Ok(update_backup::Response {}.into())
 }

+/// # `GET /_matrix/client/r0/room_keys/version`
+///
+/// Get information about the latest backup version.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/unstable/room_keys/version", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_latest_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_latest_backup::Request>,
 ) -> ConduitResult<get_latest_backup::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let (version, algorithm) =
        db.key_backups
-            .get_latest_backup(&sender_user)?
+            .get_latest_backup(sender_user)?
            .ok_or(Error::BadRequest(
                ErrorKind::NotFound,
                "Key backup does not exist.",
@@ -78,19 +86,22 @@ pub async fn get_latest_backup_route(
    .into())
 }

+/// # `GET /_matrix/client/r0/room_keys/version`
+///
+/// Get information about an existing backup.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_backup::Request<'_>>,
 ) -> ConduitResult<get_backup::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
    let algorithm = db
        .key_backups
-        .get_backup(&sender_user, &body.version)?
+        .get_backup(sender_user, &body.version)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "Key backup does not exist.",
@@ -105,50 +116,73 @@ pub async fn get_backup_route(
    .into())
 }

+/// # `DELETE /_matrix/client/r0/room_keys/version/{version}`
+///
+/// Delete an existing key backup.
+///
+/// - Deletes both information about the backup, as well as all key data related to the backup
 #[cfg_attr(
    feature = "conduit_bin",
    delete("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_backup_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_backup::Request<'_>>,
 ) -> ConduitResult<delete_backup::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    db.key_backups.delete_backup(&sender_user, &body.version)?;
+    db.key_backups.delete_backup(sender_user, &body.version)?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(delete_backup::Response.into())
+    Ok(delete_backup::Response {}.into())
 }

+/// # `PUT /_matrix/client/r0/room_keys/keys`
+///
 /// Add the received backup keys to the database.
+///
+/// - Only manipulating the most recently created version of the backup is allowed
+/// - Adds the keys to the backup
+/// - Returns the new number of keys in this backup and the etag
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/unstable/room_keys/keys", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn add_backup_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<add_backup_keys::Request<'_>>,
 ) -> ConduitResult<add_backup_keys::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

+    if Some(&body.version)
+        != db
+            .key_backups
+            .get_latest_backup_version(sender_user)?
+            .as_ref()
+    {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "You may only manipulate the most recently created version of the backup.",
+        ));
+    }
+
    for (room_id, room) in &body.rooms {
        for (session_id, key_data) in &room.sessions {
            db.key_backups.add_key(
-                &sender_user,
+                sender_user,
                &body.version,
-                &room_id,
-                &session_id,
-                &key_data,
+                room_id,
+                session_id,
+                key_data,
                &db.globals,
            )?
        }
    }

-    db.flush().await?;
+    db.flush()?;

    Ok(add_backup_keys::Response {
        count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -157,30 +191,48 @@ pub async fn add_backup_keys_route(
    .into())
 }

+/// # `PUT /_matrix/client/r0/room_keys/keys/{roomId}`
+///
 /// Add the received backup keys to the database.
+///
+/// - Only manipulating the most recently created version of the backup is allowed
+/// - Adds the keys to the backup
+/// - Returns the new number of keys in this backup and the etag
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn add_backup_key_sessions_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<add_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<add_backup_key_sessions::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

+    if Some(&body.version)
+        != db
+            .key_backups
+            .get_latest_backup_version(sender_user)?
+            .as_ref()
+    {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "You may only manipulate the most recently created version of the backup.",
+        ));
+    }
+
    for (session_id, key_data) in &body.sessions {
        db.key_backups.add_key(
-            &sender_user,
+            sender_user,
            &body.version,
            &body.room_id,
-            &session_id,
-            &key_data,
+            session_id,
+            key_data,
            &db.globals,
        )?
    }

-    db.flush().await?;
+    db.flush()?;

    Ok(add_backup_key_sessions::Response {
        count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -189,20 +241,38 @@ pub async fn add_backup_key_sessions_route(
    .into())
 }

+/// # `PUT /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}`
+///
 /// Add the received backup key to the database.
+///
+/// - Only manipulating the most recently created version of the backup is allowed
+/// - Adds the keys to the backup
+/// - Returns the new number of keys in this backup and the etag
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn add_backup_key_session_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<add_backup_key_session::Request<'_>>,
 ) -> ConduitResult<add_backup_key_session::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

+    if Some(&body.version)
+        != db
+            .key_backups
+            .get_latest_backup_version(sender_user)?
+            .as_ref()
+    {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "You may only manipulate the most recently created version of the backup.",
+        ));
+    }
+
    db.key_backups.add_key(
-        &sender_user,
+        sender_user,
        &body.version,
        &body.room_id,
        &body.session_id,
@@ -210,7 +280,7 @@ pub async fn add_backup_key_session_route(
        &db.globals,
    )?;

-    db.flush().await?;
+    db.flush()?;

    Ok(add_backup_key_session::Response {
        count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -219,79 +289,88 @@ pub async fn add_backup_key_session_route(
    .into())
 }

+/// # `GET /_matrix/client/r0/room_keys/keys`
+///
+/// Retrieves all keys from the backup.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/unstable/room_keys/keys", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_backup_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_backup_keys::Request<'_>>,
 ) -> ConduitResult<get_backup_keys::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    let rooms = db.key_backups.get_all(&sender_user, &body.version)?;
+    let rooms = db.key_backups.get_all(sender_user, &body.version)?;

    Ok(get_backup_keys::Response { rooms }.into())
 }

+/// # `GET /_matrix/client/r0/room_keys/keys/{roomId}`
+///
+/// Retrieves all keys from the backup for a given room.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_backup_key_sessions_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<get_backup_key_sessions::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let sessions = db
        .key_backups
-        .get_room(&sender_user, &body.version, &body.room_id);
+        .get_room(sender_user, &body.version, &body.room_id)?;

    Ok(get_backup_key_sessions::Response { sessions }.into())
 }

+/// # `GET /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}`
+///
+/// Retrieves a key from the backup.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_backup_key_session_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_backup_key_session::Request<'_>>,
 ) -> ConduitResult<get_backup_key_session::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let key_data = db
        .key_backups
-        .get_session(&sender_user, &body.version, &body.room_id, &body.session_id)?
-        .ok_or_else(|| {
-            Error::BadRequest(
-                ErrorKind::NotFound,
-                "Backup key not found for this user's session.",
-            )
-        })?;
+        .get_session(sender_user, &body.version, &body.room_id, &body.session_id)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "Backup key not found for this user's session.",
+        ))?;

    Ok(get_backup_key_session::Response { key_data }.into())
 }

+/// # `DELETE /_matrix/client/r0/room_keys/keys`
+///
+/// Delete the keys from the backup.
 #[cfg_attr(
    feature = "conduit_bin",
    delete("/_matrix/client/unstable/room_keys/keys", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_backup_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_backup_keys::Request<'_>>,
 ) -> ConduitResult<delete_backup_keys::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    db.key_backups
-        .delete_all_keys(&sender_user, &body.version)?;
+    db.key_backups.delete_all_keys(sender_user, &body.version)?;

-    db.flush().await?;
+    db.flush()?;

    Ok(delete_backup_keys::Response {
        count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -300,21 +379,24 @@ pub async fn delete_backup_keys_route(
    .into())
 }

+/// # `DELETE /_matrix/client/r0/room_keys/keys/{roomId}`
+///
+/// Delete the keys from the backup for a given room.
 #[cfg_attr(
    feature = "conduit_bin",
    delete("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_backup_key_sessions_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_backup_key_sessions::Request<'_>>,
 ) -> ConduitResult<delete_backup_key_sessions::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    db.key_backups
-        .delete_room_keys(&sender_user, &body.version, &body.room_id)?;
+        .delete_room_keys(sender_user, &body.version, &body.room_id)?;

-    db.flush().await?;
+    db.flush()?;

    Ok(delete_backup_key_sessions::Response {
        count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -323,21 +405,24 @@ pub async fn delete_backup_key_sessions_route(
    .into())
 }

+/// # `DELETE /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}`
+///
+/// Delete a key from the backup.
 #[cfg_attr(
    feature = "conduit_bin",
    delete("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_backup_key_session_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_backup_key_session::Request<'_>>,
 ) -> ConduitResult<delete_backup_key_session::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    db.key_backups
-        .delete_room_key(&sender_user, &body.version, &body.room_id, &body.session_id)?;
+        .delete_room_key(sender_user, &body.version, &body.room_id, &body.session_id)?;

-    db.flush().await?;
+    db.flush()?;

    Ok(delete_backup_key_session::Response {
        count: (db.key_backups.count_keys(sender_user, &body.version)? as u32).into(),
@@ -1,4 +1,4 @@
-use crate::ConduitResult;
+use crate::{ConduitResult, Ruma};
 use ruma::{
    api::client::r0::capabilities::{
        get_capabilities, Capabilities, RoomVersionStability, RoomVersionsCapability,
@@ -12,10 +12,15 @@ use rocket::get;

 /// # `GET /_matrix/client/r0/capabilities`
 ///
-/// Get information on this server's supported feature set and other relevent capabilities.
-#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/capabilities"))]
-#[tracing::instrument]
-pub async fn get_capabilities_route() -> ConduitResult<get_capabilities::Response> {
+/// Get information on the supported feature set and other relevent capabilities of this server.
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/capabilities", data = "<_body>")
+)]
+#[tracing::instrument(skip(_body))]
+pub async fn get_capabilities_route(
+    _body: Ruma<get_capabilities::Request>,
+) -> ConduitResult<get_capabilities::Response> {
    let mut available = BTreeMap::new();
    available.insert(RoomVersionId::Version5, RoomVersionStability::Stable);
    available.insert(RoomVersionId::Version6, RoomVersionStability::Stable);
@@ -1,29 +1,36 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::{
    api::client::{
        error::ErrorKind,
-        r0::config::{get_global_account_data, set_global_account_data},
+        r0::config::{
+            get_global_account_data, get_room_account_data, set_global_account_data,
+            set_room_account_data,
+        },
    },
-    events::{custom::CustomEventContent, BasicEvent},
+    events::{AnyGlobalAccountDataEventContent, AnyRoomAccountDataEventContent},
    serde::Raw,
 };
+use serde::Deserialize;
+use serde_json::{json, value::RawValue as RawJsonValue};

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};

+/// # `PUT /_matrix/client/r0/user/{userId}/account_data/{type}`
+///
+/// Sets some account data for the sender user.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/user/<_>/account_data/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn set_global_account_data_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<set_global_account_data::Request<'_>>,
 ) -> ConduitResult<set_global_account_data::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    let data = serde_json::from_str(body.data.get())
+    let data = serde_json::from_str::<serde_json::Value>(body.data.get())
        .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?;

    let event_type = body.event_type.to_string();
@@ -32,34 +39,121 @@ pub async fn set_global_account_data_route(
        None,
        sender_user,
        event_type.clone().into(),
-        &BasicEvent {
-            content: CustomEventContent { event_type, data },
-        },
+        &json!({
+            "type": event_type,
+            "content": data,
+        }),
        &db.globals,
    )?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_global_account_data::Response.into())
+    Ok(set_global_account_data::Response {}.into())
 }

+/// # `PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/account_data/{type}`
+///
+/// Sets some room account data for the sender user.
+#[cfg_attr(
+    feature = "conduit_bin",
+    put(
+        "/_matrix/client/r0/user/<_>/rooms/<_>/account_data/<_>",
+        data = "<body>"
+    )
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn set_room_account_data_route(
+    db: DatabaseGuard,
+    body: Ruma<set_room_account_data::Request<'_>>,
+) -> ConduitResult<set_room_account_data::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let data = serde_json::from_str::<serde_json::Value>(body.data.get())
+        .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Data is invalid."))?;
+
+    let event_type = body.event_type.to_string();
+
+    db.account_data.update(
+        Some(&body.room_id),
+        sender_user,
+        event_type.clone().into(),
+        &json!({
+            "type": event_type,
+            "content": data,
+        }),
+        &db.globals,
+    )?;
+
+    db.flush()?;
+
+    Ok(set_room_account_data::Response {}.into())
+}
+
+/// # `GET /_matrix/client/r0/user/{userId}/account_data/{type}`
+///
+/// Gets some account data for the sender user.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/user/<_>/account_data/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_global_account_data_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_global_account_data::Request<'_>>,
 ) -> ConduitResult<get_global_account_data::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    let data = db
+    let event = db
        .account_data
-        .get::<Raw<ruma::events::AnyBasicEvent>>(None, sender_user, body.event_type.clone().into())?
+        .get::<Box<RawJsonValue>>(None, sender_user, body.event_type.clone().into())?
        .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;

-    db.flush().await?;
+    let account_data = serde_json::from_str::<ExtractGlobalEventContent>(event.get())
+        .map_err(|_| Error::bad_database("Invalid account data event in db."))?
+        .content;

-    Ok(get_global_account_data::Response { account_data: data }.into())
+    Ok(get_global_account_data::Response { account_data }.into())
+}
+
+/// # `GET /_matrix/client/r0/user/{userId}/rooms/{roomId}/account_data/{type}`
+///
+/// Gets some room account data for the sender user.
+#[cfg_attr(
+    feature = "conduit_bin",
+    get(
+        "/_matrix/client/r0/user/<_>/rooms/<_>/account_data/<_>",
+        data = "<body>"
+    )
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_room_account_data_route(
+    db: DatabaseGuard,
+    body: Ruma<get_room_account_data::Request<'_>>,
+) -> ConduitResult<get_room_account_data::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let event = db
+        .account_data
+        .get::<Box<RawJsonValue>>(
+            Some(&body.room_id),
+            sender_user,
+            body.event_type.clone().into(),
+        )?
+        .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
+
+    let account_data = serde_json::from_str::<ExtractRoomEventContent>(event.get())
+        .map_err(|_| Error::bad_database("Invalid account data event in db."))?
+        .content;
+
+    Ok(get_room_account_data::Response { account_data }.into())
+}
+
+#[derive(Deserialize)]
+struct ExtractRoomEventContent {
+    content: Raw<AnyRoomAccountDataEventContent>,
+}
+
+#[derive(Deserialize)]
+struct ExtractGlobalEventContent {
+    content: Raw<AnyGlobalAccountDataEventContent>,
 }
@@ -1,18 +1,23 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::api::client::{error::ErrorKind, r0::context::get_context};
 use std::convert::TryFrom;

 #[cfg(feature = "conduit_bin")]
 use rocket::get;

+/// # `GET /_matrix/client/r0/rooms/{roomId}/context`
+///
+/// Allows loading room history around an event.
+///
+/// - Only works if the user is joined (TODO: always allow, but only show events if the user was
+/// joined, depending on history_visibility)
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/rooms/<_>/context/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_context_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_context::Request<'_>>,
 ) -> ConduitResult<get_context::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -24,23 +29,28 @@ pub async fn get_context_route(
        ));
    }

+    let base_pdu_id = db
+        .rooms
+        .get_pdu_id(&body.event_id)?
+        .ok_or(Error::BadRequest(
+            ErrorKind::NotFound,
+            "Base event id not found.",
+        ))?;
+
+    let base_token = db.rooms.pdu_count(&base_pdu_id)?;
+
    let base_event = db
        .rooms
-        .get_pdu(&body.event_id)?
+        .get_pdu_from_id(&base_pdu_id)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "Base event not found.",
        ))?
        .to_room_event();

-    let base_token = db
-        .rooms
-        .get_pdu_count(&body.event_id)?
-        .expect("event still exists");
-
    let events_before = db
        .rooms
-        .pdus_until(&sender_user, &body.room_id, base_token)
+        .pdus_until(sender_user, &body.room_id, base_token)?
        .take(
            u32::try_from(body.limit).map_err(|_| {
                Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.")
@@ -62,7 +72,7 @@ pub async fn get_context_route(

    let events_after = db
        .rooms
-        .pdus_after(&sender_user, &body.room_id, base_token)
+        .pdus_after(sender_user, &body.room_id, base_token)?
        .take(
            u32::try_from(body.limit).map_err(|_| {
                Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.")
@@ -1,5 +1,4 @@
-use super::State;
-use crate::{utils, ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, utils, ConduitResult, Error, Ruma};
 use ruma::api::client::{
    error::ErrorKind,
    r0::{
@@ -12,13 +11,16 @@ use super::SESSION_ID_LENGTH;
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, post, put};

+/// # `GET /_matrix/client/r0/devices`
+///
+/// Get metadata on all devices of the sender user.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/devices", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_devices_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_devices::Request>,
 ) -> ConduitResult<get_devices::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -32,58 +34,73 @@ pub async fn get_devices_route(
    Ok(get_devices::Response { devices }.into())
 }

+/// # `GET /_matrix/client/r0/devices/{deviceId}`
+///
+/// Get metadata on a single device of the sender user.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/devices/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_device_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_device::Request<'_>>,
 ) -> ConduitResult<get_device::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let device = db
        .users
-        .get_device_metadata(&sender_user, &body.body.device_id)?
+        .get_device_metadata(sender_user, &body.body.device_id)?
        .ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?;

    Ok(get_device::Response { device }.into())
 }

+/// # `PUT /_matrix/client/r0/devices/{deviceId}`
+///
+/// Updates the metadata on a given device of the sender user.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/devices/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn update_device_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<update_device::Request<'_>>,
 ) -> ConduitResult<update_device::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let mut device = db
        .users
-        .get_device_metadata(&sender_user, &body.device_id)?
+        .get_device_metadata(sender_user, &body.device_id)?
        .ok_or(Error::BadRequest(ErrorKind::NotFound, "Device not found."))?;

    device.display_name = body.display_name.clone();

    db.users
-        .update_device_metadata(&sender_user, &body.device_id, &device)?;
+        .update_device_metadata(sender_user, &body.device_id, &device)?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(update_device::Response.into())
+    Ok(update_device::Response {}.into())
 }

+/// # `PUT /_matrix/client/r0/devices/{deviceId}`
+///
+/// Deletes the given device.
+///
+/// - Requires UIAA to verify user password
+/// - Invalidates access token
+/// - Deletes device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets to-device events
+/// - Triggers device list updates
 #[cfg_attr(
    feature = "conduit_bin",
    delete("/_matrix/client/r0/devices/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_device_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_device::Request<'_>>,
 ) -> ConduitResult<delete_device::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -102,8 +119,8 @@ pub async fn delete_device_route(

    if let Some(auth) = &body.auth {
        let (worked, uiaainfo) = db.uiaa.try_auth(
-            &sender_user,
-            &sender_device,
+            sender_user,
+            sender_device,
            auth,
            &uiaainfo,
            &db.users,
@@ -113,26 +130,40 @@ pub async fn delete_device_route(
            return Err(Error::Uiaa(uiaainfo));
        }
    // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
        uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(sender_user, sender_device, &uiaainfo, &json)?;
        return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
    }

-    db.users.remove_device(&sender_user, &body.device_id)?;
+    db.users.remove_device(sender_user, &body.device_id)?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(delete_device::Response.into())
+    Ok(delete_device::Response {}.into())
 }

+/// # `PUT /_matrix/client/r0/devices/{deviceId}`
+///
+/// Deletes the given device.
+///
+/// - Requires UIAA to verify user password
+///
+/// For each device:
+/// - Invalidates access token
+/// - Deletes device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets to-device events
+/// - Triggers device list updates
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/delete_devices", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_devices_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_devices::Request<'_>>,
 ) -> ConduitResult<delete_devices::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -151,8 +182,8 @@ pub async fn delete_devices_route(

    if let Some(auth) = &body.auth {
        let (worked, uiaainfo) = db.uiaa.try_auth(
-            &sender_user,
-            &sender_device,
+            sender_user,
+            sender_device,
            auth,
            &uiaainfo,
            &db.users,
@@ -162,17 +193,20 @@ pub async fn delete_devices_route(
            return Err(Error::Uiaa(uiaainfo));
        }
    // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
        uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(sender_user, sender_device, &uiaainfo, &json)?;
        return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
    }

    for device_id in &body.devices {
-        db.users.remove_device(&sender_user, &device_id)?
+        db.users.remove_device(sender_user, device_id)?
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(delete_devices::Response.into())
+    Ok(delete_devices::Response {}.into())
 }
@@ -1,6 +1,6 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Result, Ruma};
-use log::info;
+use std::convert::TryInto;
+
+use crate::{database::DatabaseGuard, ConduitResult, Database, Error, Result, Ruma};
 use ruma::{
    api::{
        client::{
@@ -23,17 +23,23 @@ use ruma::{
    serde::Raw,
    ServerName, UInt,
 };
+use tracing::{info, warn};

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post, put};

+/// # `POST /_matrix/client/r0/publicRooms`
+///
+/// Lists the public rooms on this server.
+///
+/// - Rooms are ordered by the number of joined members
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/publicRooms", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_public_rooms_filtered_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_public_rooms_filtered::Request<'_>>,
 ) -> ConduitResult<get_public_rooms_filtered::Response> {
    get_public_rooms_filtered_helper(
@@ -47,13 +53,18 @@ pub async fn get_public_rooms_filtered_route(
    .await
 }

+/// # `GET /_matrix/client/r0/publicRooms`
+///
+/// Lists the public rooms on this server.
+///
+/// - Rooms are ordered by the number of joined members
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/publicRooms", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_public_rooms_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_public_rooms::Request<'_>>,
 ) -> ConduitResult<get_public_rooms::Response> {
    let response = get_public_rooms_filtered_helper(
@@ -76,43 +87,51 @@ pub async fn get_public_rooms_route(
    .into())
 }

+/// # `PUT /_matrix/client/r0/directory/list/room/{roomId}`
+///
+/// Sets the visibility of a given room in the room directory.
+///
+/// - TODO: Access control checks
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/directory/list/room/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn set_room_visibility_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<set_room_visibility::Request<'_>>,
 ) -> ConduitResult<set_room_visibility::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    match &body.visibility {
-        room::Visibility::_Custom(_s) => {
-            return Err(Error::BadRequest(
-                ErrorKind::InvalidParam,
-                "Room visibility type is not supported.",
-            ));
-        }
        room::Visibility::Public => {
            db.rooms.set_public(&body.room_id, true)?;
            info!("{} made {} public", sender_user, body.room_id);
        }
        room::Visibility::Private => db.rooms.set_public(&body.room_id, false)?,
+        _ => {
+            return Err(Error::BadRequest(
+                ErrorKind::InvalidParam,
+                "Room visibility type is not supported.",
+            ));
+        }
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_room_visibility::Response.into())
+    Ok(set_room_visibility::Response {}.into())
 }

+/// # `GET /_matrix/client/r0/directory/list/room/{roomId}`
+///
+/// Gets the visibility of a given room in the room directory.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/directory/list/room/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_room_visibility_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_room_visibility::Request<'_>>,
 ) -> ConduitResult<get_room_visibility::Response> {
    Ok(get_room_visibility::Response {
@@ -125,7 +144,7 @@ pub async fn get_room_visibility_route(
    .into())
 }

-pub async fn get_public_rooms_filtered_helper(
+pub(crate) async fn get_public_rooms_filtered_helper(
    db: &Database,
    server: Option<&ServerName>,
    limit: Option<UInt>,
@@ -133,9 +152,7 @@ pub async fn get_public_rooms_filtered_helper(
    filter: &IncomingFilter,
    _network: &IncomingRoomNetwork,
 ) -> ConduitResult<get_public_rooms_filtered::Response> {
-    if let Some(other_server) = server
-        .clone()
-        .filter(|server| *server != db.globals.server_name().as_str())
+    if let Some(other_server) = server.filter(|server| *server != db.globals.server_name().as_str())
    {
        let response = db
            .sending
@@ -160,15 +177,12 @@ pub async fn get_public_rooms_filtered_helper(
                .map(|c| {
                    // Convert ruma::api::federation::directory::get_public_rooms::v1::PublicRoomsChunk
                    // to ruma::api::client::r0::directory::PublicRoomsChunk
-                    Ok::<_, Error>(
-                        serde_json::from_str(
-                            &serde_json::to_string(&c)
-                                .expect("PublicRoomsChunk::to_string always works"),
-                        )
-                        .expect("federation and client-server PublicRoomsChunk are the same type"),
+                    serde_json::from_str(
+                        &serde_json::to_string(&c)
+                            .expect("PublicRoomsChunk::to_string always works"),
                    )
+                    .expect("federation and client-server PublicRoomsChunk are the same type")
                })
-                .filter_map(|r| r.ok())
                .collect(),
            prev_batch: response.prev_batch,
            next_batch: response.next_batch,
@@ -209,13 +223,11 @@ pub async fn get_public_rooms_filtered_helper(
            .map(|room_id| {
                let room_id = room_id?;

-                // TODO: Do not load full state?
-                let state = db.rooms.room_state_full(&room_id)?;
-
                let chunk = PublicRoomsChunk {
                    aliases: Vec::new(),
-                    canonical_alias: state
-                        .get(&(EventType::RoomCanonicalAlias, "".to_owned()))
+                    canonical_alias: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomCanonicalAlias, "")?
                        .map_or(Ok::<_, Error>(None), |s| {
                            Ok(serde_json::from_value::<
                                Raw<canonical_alias::CanonicalAliasEventContent>,
@@ -227,9 +239,10 @@ pub async fn get_public_rooms_filtered_helper(
                            })?
                            .alias)
                        })?,
-                    name: state.get(&(EventType::RoomName, "".to_owned())).map_or(
-                        Ok::<_, Error>(None),
-                        |s| {
+                    name: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomName, "")?
+                        .map_or(Ok::<_, Error>(None), |s| {
                            Ok(serde_json::from_value::<Raw<name::NameEventContent>>(
                                s.content.clone(),
                            )
@@ -238,15 +251,21 @@ pub async fn get_public_rooms_filtered_helper(
                            .map_err(|_| {
                                Error::bad_database("Invalid room name event in database.")
                            })?
-                            .name()
-                            .map(|n| n.to_owned()))
-                        },
-                    )?,
-                    num_joined_members: (db.rooms.room_members(&room_id).count() as u32).into(),
-                    room_id,
-                    topic: state.get(&(EventType::RoomTopic, "".to_owned())).map_or(
-                        Ok::<_, Error>(None),
-                        |s| {
+                            .name)
+                        })?,
+                    num_joined_members: db
+                        .rooms
+                        .room_joined_count(&room_id)?
+                        .unwrap_or_else(|| {
+                            warn!("Room {} has no member count", room_id);
+                            0
+                        })
+                        .try_into()
+                        .expect("user count should not be that big"),
+                    topic: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomTopic, "")?
+                        .map_or(Ok::<_, Error>(None), |s| {
                            Ok(Some(
                                serde_json::from_value::<Raw<topic::TopicEventContent>>(
                                    s.content.clone(),
@@ -258,10 +277,10 @@ pub async fn get_public_rooms_filtered_helper(
                                })?
                                .topic,
                            ))
-                        },
-                    )?,
-                    world_readable: state
-                        .get(&(EventType::RoomHistoryVisibility, "".to_owned()))
+                        })?,
+                    world_readable: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomHistoryVisibility, "")?
                        .map_or(Ok::<_, Error>(false), |s| {
                            Ok(serde_json::from_value::<
                                Raw<history_visibility::HistoryVisibilityEventContent>,
@@ -276,8 +295,9 @@ pub async fn get_public_rooms_filtered_helper(
                            .history_visibility
                                == history_visibility::HistoryVisibility::WorldReadable)
                        })?,
-                    guest_can_join: state
-                        .get(&(EventType::RoomGuestAccess, "".to_owned()))
+                    guest_can_join: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomGuestAccess, "")?
                        .map_or(Ok::<_, Error>(false), |s| {
                            Ok(
                            serde_json::from_value::<Raw<guest_access::GuestAccessEventContent>>(
@@ -292,8 +312,9 @@ pub async fn get_public_rooms_filtered_helper(
                                == guest_access::GuestAccess::CanJoin,
                        )
                        })?,
-                    avatar_url: state
-                        .get(&(EventType::RoomAvatar, "".to_owned()))
+                    avatar_url: db
+                        .rooms
+                        .room_state_get(&room_id, &EventType::RoomAvatar, "")?
                        .map(|s| {
                            Ok::<_, Error>(
                                serde_json::from_value::<Raw<avatar::AvatarEventContent>>(
@@ -310,10 +331,41 @@ pub async fn get_public_rooms_filtered_helper(
                        .transpose()?
                        // url is now an Option<String> so we must flatten
                        .flatten(),
+                    room_id,
                };
                Ok(chunk)
            })
            .filter_map(|r: Result<_>| r.ok()) // Filter out buggy rooms
+            .filter(|chunk| {
+                if let Some(query) = filter
+                    .generic_search_term
+                    .as_ref()
+                    .map(|q| q.to_lowercase())
+                {
+                    if let Some(name) = &chunk.name {
+                        if name.as_str().to_lowercase().contains(&query) {
+                            return true;
+                        }
+                    }
+
+                    if let Some(topic) = &chunk.topic {
+                        if topic.to_lowercase().contains(&query) {
+                            return true;
+                        }
+                    }
+
+                    if let Some(canonical_alias) = &chunk.canonical_alias {
+                        if canonical_alias.as_str().to_lowercase().contains(&query) {
+                            return true;
+                        }
+                    }
+
+                    false
+                } else {
+                    // No search term
+                    true
+                }
+            })
            // We need to collect all, so we can sort by member count
            .collect::<Vec<_>>();

@@ -4,6 +4,9 @@ use ruma::api::client::r0::filter::{self, create_filter, get_filter};
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};

+/// # `GET /_matrix/client/r0/user/{userId}/filter/{filterId}`
+///
+/// TODO: Loads a filter that was previously created.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))]
 #[tracing::instrument]
 pub async fn get_filter_route() -> ConduitResult<get_filter::Response> {
@@ -18,6 +21,9 @@ pub async fn get_filter_route() -> ConduitResult<get_filter::Response> {
    .into())
 }

+/// # `PUT /_matrix/client/r0/user/{userId}/filter`
+///
+/// TODO: Creates a new filter to be used by other endpoints.
 #[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/user/<_>/filter"))]
 #[tracing::instrument]
 pub async fn create_filter_route() -> ConduitResult<create_filter::Response> {
@@ -1,30 +1,42 @@
-use super::{State, SESSION_ID_LENGTH};
-use crate::{utils, ConduitResult, Database, Error, Ruma};
+use super::SESSION_ID_LENGTH;
+use crate::{database::DatabaseGuard, utils, ConduitResult, Database, Error, Result, Ruma};
+use rocket::futures::{prelude::*, stream::FuturesUnordered};
 use ruma::{
-    api::client::{
-        error::ErrorKind,
-        r0::{
-            keys::{
-                claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures,
-                upload_signing_keys,
+    api::{
+        client::{
+            error::ErrorKind,
+            r0::{
+                keys::{
+                    claim_keys, get_key_changes, get_keys, upload_keys, upload_signatures,
+                    upload_signing_keys,
+                },
+                uiaa::{AuthFlow, UiaaInfo},
            },
-            uiaa::{AuthFlow, UiaaInfo},
        },
+        federation,
    },
    encryption::UnsignedDeviceInfo,
+    DeviceId, DeviceKeyAlgorithm, UserId,
 };
-use std::collections::{BTreeMap, HashSet};
+use serde_json::json;
+use std::collections::{BTreeMap, HashMap, HashSet};

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};

+/// # `POST /_matrix/client/r0/keys/upload`
+///
+/// Publish end-to-end encryption keys for the sender device.
+///
+/// - Adds one time keys
+/// - If there are no device keys yet: Adds device keys (TODO: merge with existing keys?)
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/keys/upload", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn upload_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<upload_keys::Request>,
 ) -> ConduitResult<upload_keys::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -43,6 +55,7 @@ pub async fn upload_keys_route(
    }

    if let Some(device_keys) = &body.device_keys {
+        // TODO: merge this and the existing event?
        // This check is needed to assure that signatures are kept
        if db
            .users
@@ -59,7 +72,7 @@ pub async fn upload_keys_route(
        }
    }

-    db.flush().await?;
+    db.flush()?;

    Ok(upload_keys::Response {
        one_time_key_counts: db.users.count_one_time_keys(sender_user, sender_device)?,
@@ -67,128 +80,66 @@ pub async fn upload_keys_route(
    .into())
 }

+/// # `POST /_matrix/client/r0/keys/query`
+///
+/// Get end-to-end encryption keys for the given users.
+///
+/// - Always fetches users from other servers over federation
+/// - Gets master keys, self-signing keys, user signing keys and device keys.
+/// - The master and self-signing keys contain signatures that the user is allowed to see
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/keys/query", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_keys::Request<'_>>,
 ) -> ConduitResult<get_keys::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    let mut master_keys = BTreeMap::new();
-    let mut self_signing_keys = BTreeMap::new();
-    let mut user_signing_keys = BTreeMap::new();
-    let mut device_keys = BTreeMap::new();
+    let response = get_keys_helper(
+        Some(sender_user),
+        &body.device_keys,
+        |u| u == sender_user,
+        &db,
+    )
+    .await?;

-    for (user_id, device_ids) in &body.device_keys {
-        if device_ids.is_empty() {
-            let mut container = BTreeMap::new();
-            for device_id in db.users.all_device_ids(user_id) {
-                let device_id = device_id?;
-                if let Some(mut keys) = db.users.get_device_keys(user_id, &device_id)? {
-                    let metadata = db
-                        .users
-                        .get_device_metadata(user_id, &device_id)?
-                        .ok_or_else(|| {
-                            Error::bad_database("all_device_keys contained nonexistent device.")
-                        })?;
-
-                    keys.unsigned = UnsignedDeviceInfo {
-                        device_display_name: metadata.display_name,
-                    };
-
-                    container.insert(device_id, keys);
-                }
-            }
-            device_keys.insert(user_id.clone(), container);
-        } else {
-            for device_id in device_ids {
-                let mut container = BTreeMap::new();
-                if let Some(mut keys) = db.users.get_device_keys(&user_id.clone(), &device_id)? {
-                    let metadata = db.users.get_device_metadata(user_id, &device_id)?.ok_or(
-                        Error::BadRequest(
-                            ErrorKind::InvalidParam,
-                            "Tried to get keys for nonexistent device.",
-                        ),
-                    )?;
-
-                    keys.unsigned = UnsignedDeviceInfo {
-                        device_display_name: metadata.display_name,
-                    };
-
-                    container.insert(device_id.clone(), keys);
-                }
-                device_keys.insert(user_id.clone(), container);
-            }
-        }
-
-        if let Some(master_key) = db.users.get_master_key(user_id, sender_user)? {
-            master_keys.insert(user_id.clone(), master_key);
-        }
-        if let Some(self_signing_key) = db.users.get_self_signing_key(user_id, sender_user)? {
-            self_signing_keys.insert(user_id.clone(), self_signing_key);
-        }
-        if user_id == sender_user {
-            if let Some(user_signing_key) = db.users.get_user_signing_key(sender_user)? {
-                user_signing_keys.insert(user_id.clone(), user_signing_key);
-            }
-        }
-    }
-
-    Ok(get_keys::Response {
-        master_keys,
-        self_signing_keys,
-        user_signing_keys,
-        device_keys,
-        failures: BTreeMap::new(),
-    }
-    .into())
+    Ok(response.into())
 }

+/// # `POST /_matrix/client/r0/keys/claim`
+///
+/// Claims one-time keys
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/keys/claim", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn claim_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<claim_keys::Request>,
 ) -> ConduitResult<claim_keys::Response> {
-    let mut one_time_keys = BTreeMap::new();
-    for (user_id, map) in &body.one_time_keys {
-        let mut container = BTreeMap::new();
-        for (device_id, key_algorithm) in map {
-            if let Some(one_time_keys) =
-                db.users
-                    .take_one_time_key(user_id, device_id, key_algorithm, &db.globals)?
-            {
-                let mut c = BTreeMap::new();
-                c.insert(one_time_keys.0, one_time_keys.1);
-                container.insert(device_id.clone(), c);
-            }
-        }
-        one_time_keys.insert(user_id.clone(), container);
-    }
+    let response = claim_keys_helper(&body.one_time_keys, &db).await?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(claim_keys::Response {
-        failures: BTreeMap::new(),
-        one_time_keys,
-    }
-    .into())
+    Ok(response.into())
 }

+/// # `POST /_matrix/client/r0/keys/device_signing/upload`
+///
+/// Uploads end-to-end key information for the sender user.
+///
+/// - Requires UIAA to verify password
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/unstable/keys/device_signing/upload", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn upload_signing_keys_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<upload_signing_keys::Request<'_>>,
 ) -> ConduitResult<upload_signing_keys::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -207,8 +158,8 @@ pub async fn upload_signing_keys_route(

    if let Some(auth) = &body.auth {
        let (worked, uiaainfo) = db.uiaa.try_auth(
-            &sender_user,
-            &sender_device,
+            sender_user,
+            sender_device,
            auth,
            &uiaainfo,
            &db.users,
@@ -218,16 +169,19 @@ pub async fn upload_signing_keys_route(
            return Err(Error::Uiaa(uiaainfo));
        }
    // Success!
-    } else {
+    } else if let Some(json) = body.json_body {
        uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        db.uiaa.create(&sender_user, &sender_device, &uiaainfo)?;
+        db.uiaa
+            .create(sender_user, sender_device, &uiaainfo, &json)?;
        return Err(Error::Uiaa(uiaainfo));
+    } else {
+        return Err(Error::BadRequest(ErrorKind::NotJson, "Not json."));
    }

    if let Some(master_key) = &body.master_key {
        db.users.add_cross_signing_keys(
            sender_user,
-            &master_key,
+            master_key,
            &body.self_signing_key,
            &body.user_signing_key,
            &db.rooms,
@@ -235,18 +189,21 @@ pub async fn upload_signing_keys_route(
        )?;
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(upload_signing_keys::Response.into())
+    Ok(upload_signing_keys::Response {}.into())
 }

+/// # `POST /_matrix/client/r0/keys/signatures/upload`
+///
+/// Uploads end-to-end key signatures from the sender user.
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/unstable/keys/signatures/upload", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn upload_signatures_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<upload_signatures::Request>,
 ) -> ConduitResult<upload_signatures::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -285,10 +242,10 @@ pub async fn upload_signatures_route(
                        .to_owned(),
                );
                db.users.sign_key(
-                    &user_id,
-                    &key_id,
+                    user_id,
+                    key_id,
                    signature,
-                    &sender_user,
+                    sender_user,
                    &db.rooms,
                    &db.globals,
                )?;
@@ -296,18 +253,23 @@ pub async fn upload_signatures_route(
        }
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(upload_signatures::Response.into())
+    Ok(upload_signatures::Response {}.into())
 }

+/// # `POST /_matrix/client/r0/keys/changes`
+///
+/// Gets a list of users who have updated their device identity keys since the previous sync token.
+///
+/// - TODO: left users
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/keys/changes", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_key_changes_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_key_changes::Request<'_>>,
 ) -> ConduitResult<get_key_changes::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -351,3 +313,189 @@ pub async fn get_key_changes_route(
    }
    .into())
 }
+
+pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
+    sender_user: Option<&UserId>,
+    device_keys_input: &BTreeMap<UserId, Vec<Box<DeviceId>>>,
+    allowed_signatures: F,
+    db: &Database,
+) -> Result<get_keys::Response> {
+    let mut master_keys = BTreeMap::new();
+    let mut self_signing_keys = BTreeMap::new();
+    let mut user_signing_keys = BTreeMap::new();
+    let mut device_keys = BTreeMap::new();
+
+    let mut get_over_federation = HashMap::new();
+
+    for (user_id, device_ids) in device_keys_input {
+        if user_id.server_name() != db.globals.server_name() {
+            get_over_federation
+                .entry(user_id.server_name())
+                .or_insert_with(Vec::new)
+                .push((user_id, device_ids));
+            continue;
+        }
+
+        if device_ids.is_empty() {
+            let mut container = BTreeMap::new();
+            for device_id in db.users.all_device_ids(user_id) {
+                let device_id = device_id?;
+                if let Some(mut keys) = db.users.get_device_keys(user_id, &device_id)? {
+                    let metadata = db
+                        .users
+                        .get_device_metadata(user_id, &device_id)?
+                        .ok_or_else(|| {
+                            Error::bad_database("all_device_keys contained nonexistent device.")
+                        })?;
+
+                    keys.unsigned = UnsignedDeviceInfo {
+                        device_display_name: metadata.display_name,
+                    };
+
+                    container.insert(device_id, keys);
+                }
+            }
+            device_keys.insert(user_id.clone(), container);
+        } else {
+            for device_id in device_ids {
+                let mut container = BTreeMap::new();
+                if let Some(mut keys) = db.users.get_device_keys(&user_id.clone(), device_id)? {
+                    let metadata = db.users.get_device_metadata(user_id, device_id)?.ok_or(
+                        Error::BadRequest(
+                            ErrorKind::InvalidParam,
+                            "Tried to get keys for nonexistent device.",
+                        ),
+                    )?;
+
+                    keys.unsigned = UnsignedDeviceInfo {
+                        device_display_name: metadata.display_name,
+                    };
+
+                    container.insert(device_id.clone(), keys);
+                }
+                device_keys.insert(user_id.clone(), container);
+            }
+        }
+
+        if let Some(master_key) = db.users.get_master_key(user_id, &allowed_signatures)? {
+            master_keys.insert(user_id.clone(), master_key);
+        }
+        if let Some(self_signing_key) = db
+            .users
+            .get_self_signing_key(user_id, &allowed_signatures)?
+        {
+            self_signing_keys.insert(user_id.clone(), self_signing_key);
+        }
+        if Some(user_id) == sender_user {
+            if let Some(user_signing_key) = db.users.get_user_signing_key(user_id)? {
+                user_signing_keys.insert(user_id.clone(), user_signing_key);
+            }
+        }
+    }
+
+    let mut failures = BTreeMap::new();
+
+    let mut futures = get_over_federation
+        .into_iter()
+        .map(|(server, vec)| async move {
+            let mut device_keys_input_fed = BTreeMap::new();
+            for (user_id, keys) in vec {
+                device_keys_input_fed.insert(user_id.clone(), keys.clone());
+            }
+            (
+                server,
+                db.sending
+                    .send_federation_request(
+                        &db.globals,
+                        server,
+                        federation::keys::get_keys::v1::Request {
+                            device_keys: device_keys_input_fed,
+                        },
+                    )
+                    .await,
+            )
+        })
+        .collect::<FuturesUnordered<_>>();
+
+    while let Some((server, response)) = futures.next().await {
+        match response {
+            Ok(response) => {
+                master_keys.extend(response.master_keys);
+                self_signing_keys.extend(response.self_signing_keys);
+                device_keys.extend(response.device_keys);
+            }
+            Err(_e) => {
+                failures.insert(server.to_string(), json!({}));
+            }
+        }
+    }
+
+    Ok(get_keys::Response {
+        master_keys,
+        self_signing_keys,
+        user_signing_keys,
+        device_keys,
+        failures,
+    })
+}
+
+pub(crate) async fn claim_keys_helper(
+    one_time_keys_input: &BTreeMap<UserId, BTreeMap<Box<DeviceId>, DeviceKeyAlgorithm>>,
+    db: &Database,
+) -> Result<claim_keys::Response> {
+    let mut one_time_keys = BTreeMap::new();
+
+    let mut get_over_federation = BTreeMap::new();
+
+    for (user_id, map) in one_time_keys_input {
+        if user_id.server_name() != db.globals.server_name() {
+            get_over_federation
+                .entry(user_id.server_name())
+                .or_insert_with(Vec::new)
+                .push((user_id, map));
+        }
+
+        let mut container = BTreeMap::new();
+        for (device_id, key_algorithm) in map {
+            if let Some(one_time_keys) =
+                db.users
+                    .take_one_time_key(user_id, device_id, key_algorithm, &db.globals)?
+            {
+                let mut c = BTreeMap::new();
+                c.insert(one_time_keys.0, one_time_keys.1);
+                container.insert(device_id.clone(), c);
+            }
+        }
+        one_time_keys.insert(user_id.clone(), container);
+    }
+
+    let mut failures = BTreeMap::new();
+
+    for (server, vec) in get_over_federation {
+        let mut one_time_keys_input_fed = BTreeMap::new();
+        for (user_id, keys) in vec {
+            one_time_keys_input_fed.insert(user_id.clone(), keys.clone());
+        }
+        // Ignore failures
+        if let Ok(keys) = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                server,
+                federation::keys::claim_keys::v1::Request {
+                    one_time_keys: one_time_keys_input_fed,
+                },
+            )
+            .await
+        {
+            one_time_keys.extend(keys.one_time_keys);
+        } else {
+            failures.insert(server.to_string(), json!({}));
+        }
+    }
+
+    Ok(claim_keys::Response {
+        failures,
+        one_time_keys,
+    })
+}
@@ -1,20 +1,25 @@
-use super::State;
-use crate::{database::media::FileMeta, utils, ConduitResult, Database, Error, Ruma};
+use crate::{
+    database::{media::FileMeta, DatabaseGuard},
+    utils, ConduitResult, Error, Ruma,
+};
 use ruma::api::client::{
    error::ErrorKind,
    r0::media::{create_content, get_content, get_content_thumbnail, get_media_config},
 };
+use std::convert::TryInto;

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};
-use std::convert::TryInto;

 const MXC_LENGTH: usize = 32;

+/// # `GET /_matrix/media/r0/config`
+///
+/// Returns max upload size.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/media/r0/config"))]
 #[tracing::instrument(skip(db))]
 pub async fn get_media_config_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
 ) -> ConduitResult<get_media_config::Response> {
    Ok(get_media_config::Response {
        upload_size: db.globals.max_request_size().into(),
@@ -22,13 +27,19 @@ pub async fn get_media_config_route(
    .into())
 }

+/// # `POST /_matrix/media/r0/upload`
+///
+/// Permanently save media in the server.
+///
+/// - Some metadata will be saved in the database
+/// - Media will be saved in the media/ directory
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/media/r0/upload", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn create_content_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<create_content::Request<'_>>,
 ) -> ConduitResult<create_content::Response> {
    let mxc = format!(
@@ -36,43 +47,56 @@ pub async fn create_content_route(
        db.globals.server_name(),
        utils::random_string(MXC_LENGTH)
    );
-    db.media.create(
-        mxc.clone(),
-        &body.filename.as_deref(),
-        &body.content_type.as_deref(),
-        &body.file,
-    )?;

-    db.flush().await?;
+    db.media
+        .create(
+            mxc.clone(),
+            &db.globals,
+            &body
+                .filename
+                .as_ref()
+                .map(|filename| "inline; filename=".to_owned() + filename)
+                .as_deref(),
+            &body.content_type.as_deref(),
+            &body.file,
+        )
+        .await?;
+
+    db.flush()?;

    Ok(create_content::Response {
-        content_uri: mxc,
+        content_uri: mxc.try_into().expect("Invalid mxc:// URI"),
        blurhash: None,
    }
    .into())
 }

+/// # `POST /_matrix/media/r0/download/{serverName}/{mediaId}`
+///
+/// Load media from our server or over federation.
+///
+/// - Only allows federation if `allow_remote` is true
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/media/r0/download/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_content_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_content::Request<'_>>,
 ) -> ConduitResult<get_content::Response> {
    let mxc = format!("mxc://{}/{}", body.server_name, body.media_id);

    if let Some(FileMeta {
-        filename,
+        content_disposition,
        content_type,
        file,
-    }) = db.media.get(&mxc)?
+    }) = db.media.get(&db.globals, &mxc).await?
    {
        Ok(get_content::Response {
            file,
            content_type,
-            content_disposition: filename,
+            content_disposition,
        }
        .into())
    } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
@@ -89,12 +113,15 @@ pub async fn get_content_route(
            )
            .await?;

-        db.media.create(
-            mxc,
-            &get_content_response.content_disposition.as_deref(),
-            &get_content_response.content_type.as_deref(),
-            &get_content_response.file,
-        )?;
+        db.media
+            .create(
+                mxc,
+                &db.globals,
+                &get_content_response.content_disposition.as_deref(),
+                &get_content_response.content_type.as_deref(),
+                &get_content_response.file,
+            )
+            .await?;

        Ok(get_content_response.into())
    } else {
@@ -102,28 +129,38 @@ pub async fn get_content_route(
    }
 }

+/// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}`
+///
+/// Load media thumbnail from our server or over federation.
+///
+/// - Only allows federation if `allow_remote` is true
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/media/r0/thumbnail/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_content_thumbnail_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_content_thumbnail::Request<'_>>,
 ) -> ConduitResult<get_content_thumbnail::Response> {
    let mxc = format!("mxc://{}/{}", body.server_name, body.media_id);

    if let Some(FileMeta {
        content_type, file, ..
-    }) = db.media.get_thumbnail(
-        mxc.clone(),
-        body.width
-            .try_into()
-            .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
-        body.height
-            .try_into()
-            .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
-    )? {
+    }) = db
+        .media
+        .get_thumbnail(
+            mxc.clone(),
+            &db.globals,
+            body.width
+                .try_into()
+                .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
+            body.height
+                .try_into()
+                .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?,
+        )
+        .await?
+    {
        Ok(get_content_thumbnail::Response { file, content_type }.into())
    } else if &*body.server_name != db.globals.server_name() && body.allow_remote {
        let get_thumbnail_response = db
@@ -142,14 +179,17 @@ pub async fn get_content_thumbnail_route(
            )
            .await?;

-        db.media.upload_thumbnail(
-            mxc,
-            &None,
-            &get_thumbnail_response.content_type,
-            body.width.try_into().expect("all UInts are valid u32s"),
-            body.height.try_into().expect("all UInts are valid u32s"),
-            &get_thumbnail_response.file,
-        )?;
+        db.media
+            .upload_thumbnail(
+                mxc,
+                &db.globals,
+                &None,
+                &get_thumbnail_response.content_type,
+                body.width.try_into().expect("all UInts are valid u32s"),
+                body.height.try_into().expect("all UInts are valid u32s"),
+                &get_thumbnail_response.file,
+            )
+            .await?;

        Ok(get_thumbnail_response.into())
    } else {
@@ -1,33 +1,50 @@
-use super::State;
-use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma};
 use ruma::{
    api::client::{
        error::ErrorKind,
        r0::message::{get_message_events, send_message_event},
    },
-    events::EventContent,
+    events::EventType,
    EventId,
 };
 use std::{
    collections::BTreeMap,
    convert::{TryFrom, TryInto},
+    sync::Arc,
 };

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};

+/// # `PUT /_matrix/client/r0/rooms/{roomId}/send/{eventType}/{txnId}`
+///
+/// Send a message event into the room.
+///
+/// - Is a NOOP if the txn id was already used before and returns the same event id again
+/// - The only requirement for the content is that it has to be valid json
+/// - Tries to send the event into the room, auth rules will determine if it is allowed
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/rooms/<_>/send/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn send_message_event_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<send_message_event::Request<'_>>,
 ) -> ConduitResult<send_message_event::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
    let sender_device = body.sender_device.as_deref();

+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(body.room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
    // Check if this is a new transaction id
    if let Some(response) =
        db.transaction_ids
@@ -55,21 +72,17 @@ pub async fn send_message_event_route(

    let event_id = db.rooms.build_and_append_pdu(
        PduBuilder {
-            event_type: body.content.event_type().into(),
-            content: serde_json::from_str(
-                body.json_body
-                    .as_ref()
-                    .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?
-                    .get(),
-            )
-            .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?,
+            event_type: EventType::from(&body.event_type),
+            content: serde_json::from_str(body.body.body.json().get())
+                .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?,
            unsigned: Some(unsigned),
            state_key: None,
            redacts: None,
        },
-        &sender_user,
+        sender_user,
        &body.room_id,
        &db,
+        &state_lock,
    )?;

    db.transaction_ids.add_txnid(
@@ -79,18 +92,26 @@ pub async fn send_message_event_route(
        event_id.as_bytes(),
    )?;

-    db.flush().await?;
+    drop(state_lock);
+
+    db.flush()?;

    Ok(send_message_event::Response::new(event_id).into())
 }

+/// # `GET /_matrix/client/r0/rooms/{roomId}/messages`
+///
+/// Allows paginating through room history.
+///
+/// - Only works if the user is joined (TODO: always allow, but only show events where the user was
+/// joined, depending on history_visibility)
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/rooms/<_>/messages", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_message_events_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_message_events::Request<'_>>,
 ) -> ConduitResult<get_message_events::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -120,7 +141,7 @@ pub async fn get_message_events_route(
        get_message_events::Direction::Forward => {
            let events_after = db
                .rooms
-                .pdus_after(&sender_user, &body.room_id, from)
+                .pdus_after(sender_user, &body.room_id, from)?
                .take(limit)
                .filter_map(|r| r.ok()) // Filter out buggy events
                .filter_map(|(pdu_id, pdu)| {
@@ -150,7 +171,7 @@ pub async fn get_message_events_route(
        get_message_events::Direction::Backward => {
            let events_before = db
                .rooms
-                .pdus_until(&sender_user, &body.room_id, from)
+                .pdus_until(sender_user, &body.room_id, from)?
                .take(limit)
                .filter_map(|r| r.ok()) // Filter out buggy events
                .filter_map(|(pdu_id, pdu)| {
@@ -64,18 +64,19 @@ pub use voip::*;
 use super::State;
 #[cfg(feature = "conduit_bin")]
 use {
-    crate::ConduitResult,
-    rocket::{options, State},
-    ruma::api::client::r0::to_device::send_event_to_device,
+    crate::ConduitResult, rocket::options, ruma::api::client::r0::to_device::send_event_to_device,
 };

-const DEVICE_ID_LENGTH: usize = 10;
-const TOKEN_LENGTH: usize = 256;
-const SESSION_ID_LENGTH: usize = 256;
+pub const DEVICE_ID_LENGTH: usize = 10;
+pub const TOKEN_LENGTH: usize = 256;
+pub const SESSION_ID_LENGTH: usize = 256;

+/// # `OPTIONS`
+///
+/// Web clients use this to get CORS headers.
 #[cfg(feature = "conduit_bin")]
 #[options("/<_..>")]
 #[tracing::instrument]
 pub async fn options_route() -> ConduitResult<send_event_to_device::Response> {
-    Ok(send_event_to_device::Response.into())
+    Ok(send_event_to_device::Response {}.into())
 }
@@ -1,33 +1,35 @@
-use super::State;
-use crate::{utils, ConduitResult, Database, Ruma};
-use ruma::api::client::r0::presence::set_presence;
-use std::convert::TryInto;
+use crate::{database::DatabaseGuard, utils, ConduitResult, Ruma};
+use ruma::api::client::r0::presence::{get_presence, set_presence};
+use std::{convert::TryInto, time::Duration};

 #[cfg(feature = "conduit_bin")]
-use rocket::put;
+use rocket::{get, put};

+/// # `PUT /_matrix/client/r0/presence/{userId}/status`
+///
+/// Sets the presence state of the sender user.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/presence/<_>/status", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn set_presence_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<set_presence::Request<'_>>,
 ) -> ConduitResult<set_presence::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    for room_id in db.rooms.rooms_joined(&sender_user) {
+    for room_id in db.rooms.rooms_joined(sender_user) {
        let room_id = room_id?;

        db.rooms.edus.update_presence(
-            &sender_user,
+            sender_user,
            &room_id,
            ruma::events::presence::PresenceEvent {
                content: ruma::events::presence::PresenceEventContent {
-                    avatar_url: db.users.avatar_url(&sender_user)?,
+                    avatar_url: db.users.avatar_url(sender_user)?,
                    currently_active: None,
-                    displayname: db.users.displayname(&sender_user)?,
+                    displayname: db.users.displayname(sender_user)?,
                    last_active_ago: Some(
                        utils::millis_since_unix_epoch()
                            .try_into()
@@ -42,7 +44,58 @@ pub async fn set_presence_route(
        )?;
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_presence::Response.into())
+    Ok(set_presence::Response {}.into())
+}
+
+/// # `GET /_matrix/client/r0/presence/{userId}/status`
+///
+/// Gets the presence state of the given user.
+///
+/// - Only works if you share a room with the user
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/presence/<_>/status", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_presence_route(
+    db: DatabaseGuard,
+    body: Ruma<get_presence::Request<'_>>,
+) -> ConduitResult<get_presence::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    let mut presence_event = None;
+
+    for room_id in db
+        .rooms
+        .get_shared_rooms(vec![sender_user.clone(), body.user_id.clone()])?
+    {
+        let room_id = room_id?;
+
+        if let Some(presence) = db
+            .rooms
+            .edus
+            .get_last_presence_event(sender_user, &room_id)?
+        {
+            presence_event = Some(presence);
+            break;
+        }
+    }
+
+    if let Some(presence) = presence_event {
+        Ok(get_presence::Response {
+            // TODO: Should ruma just use the presenceeventcontent type here?
+            status_msg: presence.content.status_msg,
+            currently_active: presence.content.currently_active,
+            last_active_ago: presence
+                .content
+                .last_active_ago
+                .map(|millis| Duration::from_millis(millis.into())),
+            presence: presence.content.presence,
+        }
+        .into())
+    } else {
+        todo!();
+    }
 }
@@ -1,81 +1,107 @@
-use super::State;
-use crate::{pdu::PduBuilder, utils, ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, pdu::PduBuilder, utils, ConduitResult, Error, Ruma};
 use ruma::{
-    api::client::{
-        error::ErrorKind,
-        r0::profile::{
-            get_avatar_url, get_display_name, get_profile, set_avatar_url, set_display_name,
+    api::{
+        client::{
+            error::ErrorKind,
+            r0::profile::{
+                get_avatar_url, get_display_name, get_profile, set_avatar_url, set_display_name,
+            },
        },
+        federation::{self, query::get_profile_information::v1::ProfileField},
    },
    events::EventType,
    serde::Raw,
 };
+use std::{convert::TryInto, sync::Arc};

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
-use std::convert::TryInto;

+/// # `PUT /_matrix/client/r0/profile/{userId}/displayname`
+///
+/// Updates the displayname.
+///
+/// - Also makes sure other users receive the update using presence EDUs
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/profile/<_>/displayname", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn set_displayname_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<set_display_name::Request<'_>>,
 ) -> ConduitResult<set_display_name::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    db.users
-        .set_displayname(&sender_user, body.displayname.clone())?;
+        .set_displayname(sender_user, body.displayname.clone())?;

    // Send a new membership event and presence update into all joined rooms
-    for room_id in db.rooms.rooms_joined(&sender_user) {
-        let room_id = room_id?;
-        db.rooms.build_and_append_pdu(
-            PduBuilder {
-                event_type: EventType::RoomMember,
-                content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
-                    displayname: body.displayname.clone(),
-                    ..serde_json::from_value::<Raw<_>>(
-                        db.rooms
-                            .room_state_get(
-                                &room_id,
-                                &EventType::RoomMember,
-                                &sender_user.to_string(),
-                            )?
-                            .ok_or_else(|| {
-                                Error::bad_database(
-                                    "Tried to send displayname update for user not in the room.",
-                                )
-                            })?
-                            .1
-                            .content
-                            .clone(),
-                    )
-                    .expect("from_value::<Raw<..>> can never fail")
-                    .deserialize()
-                    .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
-                })
-                .expect("event is valid, we just created it"),
-                unsigned: None,
-                state_key: Some(sender_user.to_string()),
-                redacts: None,
-            },
-            &sender_user,
-            &room_id,
-            &db,
-        )?;
+    let all_rooms_joined: Vec<_> = db
+        .rooms
+        .rooms_joined(sender_user)
+        .filter_map(|r| r.ok())
+        .map(|room_id| {
+            Ok::<_, Error>((
+                PduBuilder {
+                    event_type: EventType::RoomMember,
+                    content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
+                        displayname: body.displayname.clone(),
+                        ..serde_json::from_value::<Raw<_>>(
+                            db.rooms
+                                .room_state_get(
+                                    &room_id,
+                                    &EventType::RoomMember,
+                                    &sender_user.to_string(),
+                                )?
+                                .ok_or_else(|| {
+                                    Error::bad_database(
+                                        "Tried to send displayname update for user not in the \
+                                     room.",
+                                    )
+                                })?
+                                .content
+                                .clone(),
+                        )
+                        .expect("from_value::<Raw<..>> can never fail")
+                        .deserialize()
+                        .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
+                    })
+                    .expect("event is valid, we just created it"),
+                    unsigned: None,
+                    state_key: Some(sender_user.to_string()),
+                    redacts: None,
+                },
+                room_id,
+            ))
+        })
+        .filter_map(|r| r.ok())
+        .collect();
+
+    for (pdu_builder, room_id) in all_rooms_joined {
+        let mutex_state = Arc::clone(
+            db.globals
+                .roomid_mutex_state
+                .write()
+                .unwrap()
+                .entry(room_id.clone())
+                .or_default(),
+        );
+        let state_lock = mutex_state.lock().await;
+
+        let _ = db
+            .rooms
+            .build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock);

        // Presence update
        db.rooms.edus.update_presence(
-            &sender_user,
+            sender_user,
            &room_id,
            ruma::events::presence::PresenceEvent {
                content: ruma::events::presence::PresenceEventContent {
-                    avatar_url: db.users.avatar_url(&sender_user)?,
+                    avatar_url: db.users.avatar_url(sender_user)?,
                    currently_active: None,
-                    displayname: db.users.displayname(&sender_user)?,
+                    displayname: db.users.displayname(sender_user)?,
                    last_active_ago: Some(
                        utils::millis_since_unix_epoch()
                            .try_into()
@@ -90,87 +116,137 @@ pub async fn set_displayname_route(
        )?;
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_display_name::Response.into())
+    Ok(set_display_name::Response {}.into())
 }

+/// # `GET /_matrix/client/r0/profile/{userId}/displayname`
+///
+/// Returns the displayname of the user.
+///
+/// - If user is on another server: Fetches displayname over federation
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/profile/<_>/displayname", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_displayname_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_display_name::Request<'_>>,
 ) -> ConduitResult<get_display_name::Response> {
+    if body.user_id.server_name() != db.globals.server_name() {
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                body.user_id.server_name(),
+                federation::query::get_profile_information::v1::Request {
+                    user_id: &body.user_id,
+                    field: Some(&ProfileField::DisplayName),
+                },
+            )
+            .await?;
+
+        return Ok(get_display_name::Response {
+            displayname: response.displayname,
+        }
+        .into());
+    }
+
    Ok(get_display_name::Response {
        displayname: db.users.displayname(&body.user_id)?,
    }
    .into())
 }

+/// # `PUT /_matrix/client/r0/profile/{userId}/avatar_url`
+///
+/// Updates the avatar_url and blurhash.
+///
+/// - Also makes sure other users receive the update using presence EDUs
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/profile/<_>/avatar_url", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn set_avatar_url_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<set_avatar_url::Request<'_>>,
 ) -> ConduitResult<set_avatar_url::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    db.users
-        .set_avatar_url(&sender_user, body.avatar_url.clone())?;
+        .set_avatar_url(sender_user, body.avatar_url.clone())?;
+
+    db.users.set_blurhash(sender_user, body.blurhash.clone())?;

    // Send a new membership event and presence update into all joined rooms
-    for room_id in db.rooms.rooms_joined(&sender_user) {
-        let room_id = room_id?;
-        db.rooms.build_and_append_pdu(
-            PduBuilder {
-                event_type: EventType::RoomMember,
-                content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
-                    avatar_url: body.avatar_url.clone(),
-                    ..serde_json::from_value::<Raw<_>>(
-                        db.rooms
-                            .room_state_get(
-                                &room_id,
-                                &EventType::RoomMember,
-                                &sender_user.to_string(),
-                            )?
-                            .ok_or_else(|| {
-                                Error::bad_database(
-                                    "Tried to send avatar url update for user not in the room.",
-                                )
-                            })?
-                            .1
-                            .content
-                            .clone(),
-                    )
-                    .expect("from_value::<Raw<..>> can never fail")
-                    .deserialize()
-                    .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
-                })
-                .expect("event is valid, we just created it"),
-                unsigned: None,
-                state_key: Some(sender_user.to_string()),
-                redacts: None,
-            },
-            &sender_user,
-            &room_id,
-            &db,
-        )?;
+    let all_joined_rooms: Vec<_> = db
+        .rooms
+        .rooms_joined(sender_user)
+        .filter_map(|r| r.ok())
+        .map(|room_id| {
+            Ok::<_, Error>((
+                PduBuilder {
+                    event_type: EventType::RoomMember,
+                    content: serde_json::to_value(ruma::events::room::member::MemberEventContent {
+                        avatar_url: body.avatar_url.clone(),
+                        ..serde_json::from_value::<Raw<_>>(
+                            db.rooms
+                                .room_state_get(
+                                    &room_id,
+                                    &EventType::RoomMember,
+                                    &sender_user.to_string(),
+                                )?
+                                .ok_or_else(|| {
+                                    Error::bad_database(
+                                        "Tried to send displayname update for user not in the \
+                                     room.",
+                                    )
+                                })?
+                                .content
+                                .clone(),
+                        )
+                        .expect("from_value::<Raw<..>> can never fail")
+                        .deserialize()
+                        .map_err(|_| Error::bad_database("Database contains invalid PDU."))?
+                    })
+                    .expect("event is valid, we just created it"),
+                    unsigned: None,
+                    state_key: Some(sender_user.to_string()),
+                    redacts: None,
+                },
+                room_id,
+            ))
+        })
+        .filter_map(|r| r.ok())
+        .collect();
+
+    for (pdu_builder, room_id) in all_joined_rooms {
+        let mutex_state = Arc::clone(
+            db.globals
+                .roomid_mutex_state
+                .write()
+                .unwrap()
+                .entry(room_id.clone())
+                .or_default(),
+        );
+        let state_lock = mutex_state.lock().await;
+
+        let _ = db
+            .rooms
+            .build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock);

        // Presence update
        db.rooms.edus.update_presence(
-            &sender_user,
+            sender_user,
            &room_id,
            ruma::events::presence::PresenceEvent {
                content: ruma::events::presence::PresenceEventContent {
-                    avatar_url: db.users.avatar_url(&sender_user)?,
+                    avatar_url: db.users.avatar_url(sender_user)?,
                    currently_active: None,
-                    displayname: db.users.displayname(&sender_user)?,
+                    displayname: db.users.displayname(sender_user)?,
                    last_active_ago: Some(
                        utils::millis_since_unix_epoch()
                            .try_into()
@@ -185,35 +261,87 @@ pub async fn set_avatar_url_route(
        )?;
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_avatar_url::Response.into())
+    Ok(set_avatar_url::Response {}.into())
 }

+/// # `GET /_matrix/client/r0/profile/{userId}/avatar_url`
+///
+/// Returns the avatar_url and blurhash of the user.
+///
+/// - If user is on another server: Fetches avatar_url and blurhash over federation
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/profile/<_>/avatar_url", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_avatar_url_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_avatar_url::Request<'_>>,
 ) -> ConduitResult<get_avatar_url::Response> {
+    if body.user_id.server_name() != db.globals.server_name() {
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                body.user_id.server_name(),
+                federation::query::get_profile_information::v1::Request {
+                    user_id: &body.user_id,
+                    field: Some(&ProfileField::AvatarUrl),
+                },
+            )
+            .await?;
+
+        return Ok(get_avatar_url::Response {
+            avatar_url: response.avatar_url,
+            blurhash: response.blurhash,
+        }
+        .into());
+    }
+
    Ok(get_avatar_url::Response {
        avatar_url: db.users.avatar_url(&body.user_id)?,
+        blurhash: db.users.blurhash(&body.user_id)?,
    }
    .into())
 }

+/// # `GET /_matrix/client/r0/profile/{userId}`
+///
+/// Returns the displayname, avatar_url and blurhash of the user.
+///
+/// - If user is on another server: Fetches profile over federation
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/profile/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_profile_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_profile::Request<'_>>,
 ) -> ConduitResult<get_profile::Response> {
+    if body.user_id.server_name() != db.globals.server_name() {
+        let response = db
+            .sending
+            .send_federation_request(
+                &db.globals,
+                body.user_id.server_name(),
+                federation::query::get_profile_information::v1::Request {
+                    user_id: &body.user_id,
+                    field: None,
+                },
+            )
+            .await?;
+
+        return Ok(get_profile::Response {
+            displayname: response.displayname,
+            avatar_url: response.avatar_url,
+            blurhash: response.blurhash,
+        }
+        .into());
+    }
+
    if !db.users.exists(&body.user_id)? {
        // Return 404 if this user doesn't exist
        return Err(Error::BadRequest(
@@ -224,6 +352,7 @@ pub async fn get_profile_route(

    Ok(get_profile::Response {
        avatar_url: db.users.avatar_url(&body.user_id)?,
+        blurhash: db.users.blurhash(&body.user_id)?,
        displayname: db.users.displayname(&body.user_id)?,
    }
    .into())
@@ -1,37 +1,37 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::{
    api::client::{
        error::ErrorKind,
        r0::push::{
            delete_pushrule, get_pushers, get_pushrule, get_pushrule_actions, get_pushrule_enabled,
-            get_pushrules_all, set_pushrule, set_pushrule_actions, set_pushrule_enabled, RuleKind,
+            get_pushrules_all, set_pusher, set_pushrule, set_pushrule_actions,
+            set_pushrule_enabled, RuleKind,
        },
    },
-    events::EventType,
-    push::{
-        ConditionalPushRuleInit, ContentPushRule, OverridePushRule, PatternedPushRuleInit,
-        RoomPushRule, SenderPushRule, SimplePushRuleInit, UnderridePushRule,
-    },
+    events::{push_rules, EventType},
+    push::{ConditionalPushRuleInit, PatternedPushRuleInit, SimplePushRuleInit},
 };

 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, post, put};

+/// # `GET /_matrix/client/r0/pushrules`
+///
+/// Retrieves the push rules event for this user.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/pushrules", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_pushrules_all_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_pushrules_all::Request>,
 ) -> ConduitResult<get_pushrules_all::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let event = db
        .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, sender_user, EventType::PushRules)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "PushRules event not found.",
@@ -43,20 +43,23 @@ pub async fn get_pushrules_all_route(
    .into())
 }

+/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
+///
+/// Retrieves a single specified push rule for this user.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_pushrule_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_pushrule::Request<'_>>,
 ) -> ConduitResult<get_pushrule::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let event = db
        .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, sender_user, EventType::PushRules)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "PushRules event not found.",
@@ -66,30 +69,25 @@ pub async fn get_pushrule_route(
    let rule = match body.kind {
        RuleKind::Override => global
            .override_
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.clone().into()),
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
        RuleKind::Underride => global
            .underride
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.clone().into()),
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
        RuleKind::Sender => global
            .sender
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.clone().into()),
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
        RuleKind::Room => global
            .room
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.clone().into()),
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
        RuleKind::Content => global
            .content
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.clone().into()),
-        RuleKind::_Custom(_) => None,
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.clone().into()),
+        _ => None,
    };

    if let Some(rule) = rule {
@@ -102,16 +100,20 @@ pub async fn get_pushrule_route(
    }
 }

+/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
+///
+/// Creates a single specified push rule for this user.
 #[cfg_attr(
    feature = "conduit_bin",
-    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
+    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<req>")
 )]
-#[tracing::instrument(skip(db, body))]
+#[tracing::instrument(skip(db, req))]
 pub async fn set_pushrule_route(
-    db: State<'_, Database>,
-    body: Ruma<set_pushrule::Request<'_>>,
+    db: DatabaseGuard,
+    req: Ruma<set_pushrule::Request<'_>>,
 ) -> ConduitResult<set_pushrule::Response> {
-    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+    let sender_user = req.sender_user.as_ref().expect("user is authenticated");
+    let body = req.body;

    if body.scope != "global" {
        return Err(Error::BadRequest(
@@ -122,7 +124,7 @@ pub async fn set_pushrule_route(

    let mut event = db
        .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, sender_user, EventType::PushRules)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "PushRules event not found.",
@@ -131,131 +133,84 @@ pub async fn set_pushrule_route(
    let global = &mut event.content.global;
    match body.kind {
        RuleKind::Override => {
-            if let Some(rule) = global
-                .override_
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.override_.remove(&rule);
-            }
-
-            global.override_.insert(OverridePushRule(
+            global.override_.replace(
                ConditionalPushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                    default: false,
                    enabled: true,
-                    rule_id: body.rule_id.clone(),
-                    conditions: body.conditions.clone(),
+                    rule_id: body.rule_id,
+                    conditions: body.conditions,
                }
                .into(),
-            ));
+            );
        }
        RuleKind::Underride => {
-            if let Some(rule) = global
-                .underride
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.underride.remove(&rule);
-            }
-
-            global.underride.insert(UnderridePushRule(
+            global.underride.replace(
                ConditionalPushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                    default: false,
                    enabled: true,
-                    rule_id: body.rule_id.clone(),
-                    conditions: body.conditions.clone(),
+                    rule_id: body.rule_id,
+                    conditions: body.conditions,
                }
                .into(),
-            ));
+            );
        }
        RuleKind::Sender => {
-            if let Some(rule) = global
-                .sender
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.sender.remove(&rule);
-            }
-
-            global.sender.insert(SenderPushRule(
+            global.sender.replace(
                SimplePushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                    default: false,
                    enabled: true,
-                    rule_id: body.rule_id.clone(),
+                    rule_id: body.rule_id,
                }
                .into(),
-            ));
+            );
        }
        RuleKind::Room => {
-            if let Some(rule) = global
-                .room
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.room.remove(&rule);
-            }
-
-            global.room.insert(RoomPushRule(
+            global.room.replace(
                SimplePushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                    default: false,
                    enabled: true,
-                    rule_id: body.rule_id.clone(),
+                    rule_id: body.rule_id,
                }
                .into(),
-            ));
+            );
        }
        RuleKind::Content => {
-            if let Some(rule) = global
-                .content
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.content.remove(&rule);
-            }
-
-            global.content.insert(ContentPushRule(
+            global.content.replace(
                PatternedPushRuleInit {
-                    actions: body.actions.clone(),
+                    actions: body.actions,
                    default: false,
                    enabled: true,
-                    rule_id: body.rule_id.clone(),
-                    pattern: body.pattern.clone().unwrap_or_default(),
+                    rule_id: body.rule_id,
+                    pattern: body.pattern.unwrap_or_default(),
                }
                .into(),
-            ));
+            );
        }
-        RuleKind::_Custom(_) => {}
+        _ => {}
    }

-    db.account_data.update(
-        None,
-        &sender_user,
-        EventType::PushRules,
-        &event,
-        &db.globals,
-    )?;
+    db.account_data
+        .update(None, sender_user, EventType::PushRules, &event, &db.globals)?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_pushrule::Response.into())
+    Ok(set_pushrule::Response {}.into())
 }

+/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/actions`
+///
+/// Gets the actions of a single specified push rule for this user.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_pushrule_actions_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_pushrule_actions::Request<'_>>,
 ) -> ConduitResult<get_pushrule_actions::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -269,7 +224,7 @@ pub async fn get_pushrule_actions_route(

    let mut event = db
        .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, sender_user, EventType::PushRules)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "PushRules event not found.",
@@ -279,33 +234,28 @@ pub async fn get_pushrule_actions_route(
    let actions = match body.kind {
        RuleKind::Override => global
            .override_
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.actions.clone()),
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
        RuleKind::Underride => global
            .underride
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.actions.clone()),
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
        RuleKind::Sender => global
            .sender
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.actions.clone()),
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
        RuleKind::Room => global
            .room
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.actions.clone()),
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
        RuleKind::Content => global
            .content
-            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map(|rule| rule.0.actions.clone()),
-        RuleKind::_Custom(_) => None,
+            .get(body.rule_id.as_str())
+            .map(|rule| rule.actions.clone()),
+        _ => None,
    };

-    db.flush().await?;
+    db.flush()?;

    Ok(get_pushrule_actions::Response {
        actions: actions.unwrap_or_default(),
@@ -313,13 +263,16 @@ pub async fn get_pushrule_actions_route(
    .into())
 }

+/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/actions`
+///
+/// Sets the actions of a single specified push rule for this user.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn set_pushrule_actions_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<set_pushrule_actions::Request<'_>>,
 ) -> ConduitResult<set_pushrule_actions::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -333,7 +286,7 @@ pub async fn set_pushrule_actions_route(

    let mut event = db
        .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, sender_user, EventType::PushRules)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "PushRules event not found.",
@@ -342,88 +295,56 @@ pub async fn set_pushrule_actions_route(
    let global = &mut event.content.global;
    match body.kind {
        RuleKind::Override => {
-            if let Some(mut rule) = global
-                .override_
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.override_.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.override_.insert(rule);
+            if let Some(mut rule) = global.override_.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.override_.replace(rule);
            }
        }
        RuleKind::Underride => {
-            if let Some(mut rule) = global
-                .underride
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.underride.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.underride.insert(rule);
+            if let Some(mut rule) = global.underride.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.underride.replace(rule);
            }
        }
        RuleKind::Sender => {
-            if let Some(mut rule) = global
-                .sender
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.sender.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.sender.insert(rule);
+            if let Some(mut rule) = global.sender.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.sender.replace(rule);
            }
        }
        RuleKind::Room => {
-            if let Some(mut rule) = global
-                .room
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.room.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.room.insert(rule);
+            if let Some(mut rule) = global.room.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.room.replace(rule);
            }
        }
        RuleKind::Content => {
-            if let Some(mut rule) = global
-                .content
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
-                global.content.remove(&rule);
-                rule.0.actions = body.actions.clone();
-                global.content.insert(rule);
+            if let Some(mut rule) = global.content.get(body.rule_id.as_str()).cloned() {
+                rule.actions = body.actions.clone();
+                global.content.replace(rule);
            }
        }
-        RuleKind::_Custom(_) => {}
+        _ => {}
    };

-    db.account_data.update(
-        None,
-        &sender_user,
-        EventType::PushRules,
-        &event,
-        &db.globals,
-    )?;
+    db.account_data
+        .update(None, sender_user, EventType::PushRules, &event, &db.globals)?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_pushrule_actions::Response.into())
+    Ok(set_pushrule_actions::Response {}.into())
 }

+/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/enabled`
+///
+/// Gets the enabled status of a single specified push rule for this user.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_pushrule_enabled_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_pushrule_enabled::Request<'_>>,
 ) -> ConduitResult<get_pushrule_enabled::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -437,7 +358,7 @@ pub async fn get_pushrule_enabled_route(

    let mut event = db
        .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, sender_user, EventType::PushRules)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "PushRules event not found.",
@@ -448,43 +369,46 @@ pub async fn get_pushrule_enabled_route(
        RuleKind::Override => global
            .override_
            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
        RuleKind::Underride => global
            .underride
            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
        RuleKind::Sender => global
            .sender
            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
        RuleKind::Room => global
            .room
            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
        RuleKind::Content => global
            .content
            .iter()
-            .find(|rule| rule.0.rule_id == body.rule_id)
-            .map_or(false, |rule| rule.0.enabled),
-        RuleKind::_Custom(_) => false,
+            .find(|rule| rule.rule_id == body.rule_id)
+            .map_or(false, |rule| rule.enabled),
+        _ => false,
    };

-    db.flush().await?;
+    db.flush()?;

    Ok(get_pushrule_enabled::Response { enabled }.into())
 }

+/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/enabled`
+///
+/// Sets the enabled status of a single specified push rule for this user.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn set_pushrule_enabled_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<set_pushrule_enabled::Request<'_>>,
 ) -> ConduitResult<set_pushrule_enabled::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -498,7 +422,7 @@ pub async fn set_pushrule_enabled_route(

    let mut event = db
        .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<ruma::events::push_rules::PushRulesEvent>(None, sender_user, EventType::PushRules)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "PushRules event not found.",
@@ -507,88 +431,61 @@ pub async fn set_pushrule_enabled_route(
    let global = &mut event.content.global;
    match body.kind {
        RuleKind::Override => {
-            if let Some(mut rule) = global
-                .override_
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(mut rule) = global.override_.get(body.rule_id.as_str()).cloned() {
                global.override_.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                global.override_.insert(rule);
            }
        }
        RuleKind::Underride => {
-            if let Some(mut rule) = global
-                .underride
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(mut rule) = global.underride.get(body.rule_id.as_str()).cloned() {
                global.underride.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                global.underride.insert(rule);
            }
        }
        RuleKind::Sender => {
-            if let Some(mut rule) = global
-                .sender
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(mut rule) = global.sender.get(body.rule_id.as_str()).cloned() {
                global.sender.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                global.sender.insert(rule);
            }
        }
        RuleKind::Room => {
-            if let Some(mut rule) = global
-                .room
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(mut rule) = global.room.get(body.rule_id.as_str()).cloned() {
                global.room.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                global.room.insert(rule);
            }
        }
        RuleKind::Content => {
-            if let Some(mut rule) = global
-                .content
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(mut rule) = global.content.get(body.rule_id.as_str()).cloned() {
                global.content.remove(&rule);
-                rule.0.enabled = body.enabled;
+                rule.enabled = body.enabled;
                global.content.insert(rule);
            }
        }
-        RuleKind::_Custom(_) => {}
+        _ => {}
    }

-    db.account_data.update(
-        None,
-        &sender_user,
-        EventType::PushRules,
-        &event,
-        &db.globals,
-    )?;
+    db.account_data
+        .update(None, sender_user, EventType::PushRules, &event, &db.globals)?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_pushrule_enabled::Response.into())
+    Ok(set_pushrule_enabled::Response {}.into())
 }

+/// # `DELETE /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
+///
+/// Deletes a single specified push rule for this user.
 #[cfg_attr(
    feature = "conduit_bin",
    delete("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_pushrule_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_pushrule::Request<'_>>,
 ) -> ConduitResult<delete_pushrule::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -602,7 +499,7 @@ pub async fn delete_pushrule_route(

    let mut event = db
        .account_data
-        .get::<ruma::events::push_rules::PushRulesEvent>(None, &sender_user, EventType::PushRules)?
+        .get::<push_rules::PushRulesEvent>(None, sender_user, EventType::PushRules)?
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "PushRules event not found.",
@@ -611,87 +508,81 @@ pub async fn delete_pushrule_route(
    let global = &mut event.content.global;
    match body.kind {
        RuleKind::Override => {
-            if let Some(rule) = global
-                .override_
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(rule) = global.override_.get(body.rule_id.as_str()).cloned() {
                global.override_.remove(&rule);
            }
        }
        RuleKind::Underride => {
-            if let Some(rule) = global
-                .underride
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(rule) = global.underride.get(body.rule_id.as_str()).cloned() {
                global.underride.remove(&rule);
            }
        }
        RuleKind::Sender => {
-            if let Some(rule) = global
-                .sender
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(rule) = global.sender.get(body.rule_id.as_str()).cloned() {
                global.sender.remove(&rule);
            }
        }
        RuleKind::Room => {
-            if let Some(rule) = global
-                .room
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(rule) = global.room.get(body.rule_id.as_str()).cloned() {
                global.room.remove(&rule);
            }
        }
        RuleKind::Content => {
-            if let Some(rule) = global
-                .content
-                .iter()
-                .find(|rule| rule.0.rule_id == body.rule_id)
-                .cloned()
-            {
+            if let Some(rule) = global.content.get(body.rule_id.as_str()).cloned() {
                global.content.remove(&rule);
            }
        }
-        RuleKind::_Custom(_) => {}
+        _ => {}
    }

-    db.account_data.update(
-        None,
-        &sender_user,
-        EventType::PushRules,
-        &event,
-        &db.globals,
-    )?;
+    db.account_data
+        .update(None, sender_user, EventType::PushRules, &event, &db.globals)?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(delete_pushrule::Response.into())
+    Ok(delete_pushrule::Response {}.into())
 }

-#[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/pushers"))]
-#[tracing::instrument]
-pub async fn get_pushers_route() -> ConduitResult<get_pushers::Response> {
+/// # `GET /_matrix/client/r0/pushers`
+///
+/// Gets all currently active pushers for the sender user.
+#[cfg_attr(
+    feature = "conduit_bin",
+    get("/_matrix/client/r0/pushers", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_pushers_route(
+    db: DatabaseGuard,
+    body: Ruma<get_pushers::Request>,
+) -> ConduitResult<get_pushers::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
    Ok(get_pushers::Response {
-        pushers: Vec::new(),
+        pushers: db.pusher.get_pushers(sender_user)?,
    }
    .into())
 }

-#[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/pushers/set"))]
-#[tracing::instrument(skip(db))]
-pub async fn set_pushers_route(db: State<'_, Database>) -> ConduitResult<get_pushers::Response> {
-    db.flush().await?;
+/// # `POST /_matrix/client/r0/pushers/set`
+///
+/// Adds a pusher for the sender user.
+///
+/// - TODO: Handle `append`
+#[cfg_attr(
+    feature = "conduit_bin",
+    post("/_matrix/client/r0/pushers/set", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn set_pushers_route(
+    db: DatabaseGuard,
+    body: Ruma<set_pusher::Request>,
+) -> ConduitResult<set_pusher::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+    let pusher = body.pusher.clone();

-    Ok(get_pushers::Response {
-        pushers: Vec::new(),
-    }
-    .into())
+    db.pusher.set_pusher(sender_user, pusher)?;
+
+    db.flush()?;
+
+    Ok(set_pusher::Response::default().into())
 }
@@ -1,24 +1,31 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::{
    api::client::{
        error::ErrorKind,
        r0::{read_marker::set_read_marker, receipt::create_receipt},
    },
-    events::{AnyEphemeralRoomEvent, AnyEvent, EventType},
+    events::{AnyEphemeralRoomEvent, EventType},
+    receipt::ReceiptType,
+    MilliSecondsSinceUnixEpoch,
 };
+use std::collections::BTreeMap;

 #[cfg(feature = "conduit_bin")]
 use rocket::post;
-use std::{collections::BTreeMap, time::SystemTime};

+/// # `POST /_matrix/client/r0/rooms/{roomId}/read_markers`
+///
+/// Sets different types of read markers.
+///
+/// - Updates fully-read account data event to `fully_read`
+/// - If `read_receipt` is set: Update private marker and public read receipt EDU
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/rooms/<_>/read_markers", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn set_read_marker_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<set_read_marker::Request<'_>>,
 ) -> ConduitResult<set_read_marker::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -27,11 +34,10 @@ pub async fn set_read_marker_route(
        content: ruma::events::fully_read::FullyReadEventContent {
            event_id: body.fully_read.clone(),
        },
-        room_id: body.room_id.clone(),
    };
    db.account_data.update(
        Some(&body.room_id),
-        &sender_user,
+        sender_user,
        EventType::FullyRead,
        &fully_read_event,
        &db.globals,
@@ -40,61 +46,63 @@ pub async fn set_read_marker_route(
    if let Some(event) = &body.read_receipt {
        db.rooms.edus.private_read_set(
            &body.room_id,
-            &sender_user,
+            sender_user,
            db.rooms.get_pdu_count(event)?.ok_or(Error::BadRequest(
                ErrorKind::InvalidParam,
                "Event does not exist.",
            ))?,
            &db.globals,
        )?;
+        db.rooms
+            .reset_notification_counts(sender_user, &body.room_id)?;

        let mut user_receipts = BTreeMap::new();
        user_receipts.insert(
            sender_user.clone(),
            ruma::events::receipt::Receipt {
-                ts: Some(SystemTime::now()),
-            },
-        );
-        let mut receipt_content = BTreeMap::new();
-        receipt_content.insert(
-            event.to_owned(),
-            ruma::events::receipt::Receipts {
-                read: Some(user_receipts),
+                ts: Some(MilliSecondsSinceUnixEpoch::now()),
            },
        );

+        let mut receipts = BTreeMap::new();
+        receipts.insert(ReceiptType::Read, user_receipts);
+
+        let mut receipt_content = BTreeMap::new();
+        receipt_content.insert(event.to_owned(), receipts);
+
        db.rooms.edus.readreceipt_update(
-            &sender_user,
+            sender_user,
            &body.room_id,
-            AnyEvent::Ephemeral(AnyEphemeralRoomEvent::Receipt(
-                ruma::events::receipt::ReceiptEvent {
-                    content: ruma::events::receipt::ReceiptEventContent(receipt_content),
-                    room_id: body.room_id.clone(),
-                },
-            )),
+            AnyEphemeralRoomEvent::Receipt(ruma::events::receipt::ReceiptEvent {
+                content: ruma::events::receipt::ReceiptEventContent(receipt_content),
+                room_id: body.room_id.clone(),
+            }),
            &db.globals,
        )?;
    }

-    db.flush().await?;
+    db.flush()?;

-    Ok(set_read_marker::Response.into())
+    Ok(set_read_marker::Response {}.into())
 }

+/// # `POST /_matrix/client/r0/rooms/{roomId}/receipt/{receiptType}/{eventId}`
+///
+/// Sets private read marker and public read receipt EDU.
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/rooms/<_>/receipt/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn create_receipt_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<create_receipt::Request<'_>>,
 ) -> ConduitResult<create_receipt::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    db.rooms.edus.private_read_set(
        &body.room_id,
-        &sender_user,
+        sender_user,
        db.rooms
            .get_pdu_count(&body.event_id)?
            .ok_or(Error::BadRequest(
@@ -103,35 +111,33 @@ pub async fn create_receipt_route(
            ))?,
        &db.globals,
    )?;
+    db.rooms
+        .reset_notification_counts(sender_user, &body.room_id)?;

    let mut user_receipts = BTreeMap::new();
    user_receipts.insert(
        sender_user.clone(),
        ruma::events::receipt::Receipt {
-            ts: Some(SystemTime::now()),
+            ts: Some(MilliSecondsSinceUnixEpoch::now()),
        },
    );
+    let mut receipts = BTreeMap::new();
+    receipts.insert(ReceiptType::Read, user_receipts);
+
    let mut receipt_content = BTreeMap::new();
-    receipt_content.insert(
-        body.event_id.to_owned(),
-        ruma::events::receipt::Receipts {
-            read: Some(user_receipts),
-        },
-    );
+    receipt_content.insert(body.event_id.to_owned(), receipts);

    db.rooms.edus.readreceipt_update(
-        &sender_user,
+        sender_user,
        &body.room_id,
-        AnyEvent::Ephemeral(AnyEphemeralRoomEvent::Receipt(
-            ruma::events::receipt::ReceiptEvent {
-                content: ruma::events::receipt::ReceiptEventContent(receipt_content),
-                room_id: body.room_id.clone(),
-            },
-        )),
+        AnyEphemeralRoomEvent::Receipt(ruma::events::receipt::ReceiptEvent {
+            content: ruma::events::receipt::ReceiptEventContent(receipt_content),
+            room_id: body.room_id.clone(),
+        }),
        &db.globals,
    )?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(create_receipt::Response.into())
+    Ok(create_receipt::Response {}.into())
 }
@@ -1,5 +1,6 @@
-use super::State;
-use crate::{pdu::PduBuilder, ConduitResult, Database, Ruma};
+use std::sync::Arc;
+
+use crate::{database::DatabaseGuard, pdu::PduBuilder, ConduitResult, Ruma};
 use ruma::{
    api::client::r0::redact::redact_event,
    events::{room::redaction, EventType},
@@ -8,17 +9,32 @@ use ruma::{
 #[cfg(feature = "conduit_bin")]
 use rocket::put;

+/// # `PUT /_matrix/client/r0/rooms/{roomId}/redact/{eventId}/{txnId}`
+///
+/// Tries to send a redaction event into the room.
+///
+/// - TODO: Handle txn id
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/rooms/<_>/redact/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn redact_event_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<redact_event::Request<'_>>,
 ) -> ConduitResult<redact_event::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(body.room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
    let event_id = db.rooms.build_and_append_pdu(
        PduBuilder {
            event_type: EventType::RoomRedaction,
@@ -30,12 +46,15 @@ pub async fn redact_event_route(
            state_key: None,
            redacts: Some(body.event_id.clone()),
        },
-        &sender_user,
+        sender_user,
        &body.room_id,
        &db,
+        &state_lock,
    )?;

-    db.flush().await?;
+    drop(state_lock);
+
+    db.flush()?;

    Ok(redact_event::Response { event_id }.into())
 }
@@ -1,10 +1,11 @@
-use super::State;
-use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Ruma};
-use log::info;
+use crate::{
+    client_server::invite_helper, database::DatabaseGuard, pdu::PduBuilder, ConduitResult, Error,
+    Ruma,
+};
 use ruma::{
    api::client::{
        error::ErrorKind,
-        r0::room::{self, create_room, get_room_event, upgrade_room},
+        r0::room::{self, aliases, create_room, get_room_event, upgrade_room},
    },
    events::{
        room::{guest_access, history_visibility, join_rules, member, name, topic},
@@ -13,47 +14,88 @@ use ruma::{
    serde::Raw,
    RoomAliasId, RoomId, RoomVersionId,
 };
-use std::{cmp::max, collections::BTreeMap, convert::TryFrom};
+use std::{cmp::max, collections::BTreeMap, convert::TryFrom, sync::Arc};
+use tracing::{info, warn};

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};

+/// # `POST /_matrix/client/r0/createRoom`
+///
+/// Creates a new room.
+///
+/// - Room ID is randomly generated
+/// - Create alias if room_alias_name is set
+/// - Send create event
+/// - Join sender user
+/// - Send power levels event
+/// - Send canonical room alias
+/// - Send join rules
+/// - Send history visibility
+/// - Send guest access
+/// - Send events listed in initial state
+/// - Send events implied by `name` and `topic`
+/// - Send invite events
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/createRoom", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn create_room_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<create_room::Request<'_>>,
 ) -> ConduitResult<create_room::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let room_id = RoomId::new(db.globals.server_name());

-    let alias = body
-        .room_alias_name
-        .as_ref()
-        .map_or(Ok(None), |localpart| {
-            // TODO: Check for invalid characters and maximum length
-            let alias =
-                RoomAliasId::try_from(format!("#{}:{}", localpart, db.globals.server_name()))
-                    .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?;
+    db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;

-            if db.rooms.id_from_alias(&alias)?.is_some() {
-                Err(Error::BadRequest(
-                    ErrorKind::RoomInUse,
-                    "Room alias already exists.",
-                ))
-            } else {
-                Ok(Some(alias))
-            }
-        })?;
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
+    let alias: Option<RoomAliasId> =
+        body.room_alias_name
+            .as_ref()
+            .map_or(Ok(None), |localpart| {
+                // TODO: Check for invalid characters and maximum length
+                let alias =
+                    RoomAliasId::try_from(format!("#{}:{}", localpart, db.globals.server_name()))
+                        .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?;
+
+                if db.rooms.id_from_alias(&alias)?.is_some() {
+                    Err(Error::BadRequest(
+                        ErrorKind::RoomInUse,
+                        "Room alias already exists.",
+                    ))
+                } else {
+                    Ok(Some(alias))
+                }
+            })?;

    let mut content = ruma::events::room::create::CreateEventContent::new(sender_user.clone());
    content.federate = body.creation_content.federate;
    content.predecessor = body.creation_content.predecessor.clone();
-    content.room_version = RoomVersionId::Version6;
+    content.room_version = match body.room_version.clone() {
+        Some(room_version) => {
+            if room_version == RoomVersionId::Version5 || room_version == RoomVersionId::Version6 {
+                room_version
+            } else {
+                return Err(Error::BadRequest(
+                    ErrorKind::UnsupportedRoomVersion,
+                    "This server does not support that room version.",
+                ));
+            }
+        }
+        None => RoomVersionId::Version6,
+    };

    // 1. The room create event
    db.rooms.build_and_append_pdu(
@@ -64,9 +106,10 @@ pub async fn create_room_route(
            state_key: Some("".to_owned()),
            redacts: None,
        },
-        &sender_user,
+        sender_user,
        &room_id,
        &db,
+        &state_lock,
    )?;

    // 2. Let the room creator join
@@ -75,63 +118,25 @@ pub async fn create_room_route(
            event_type: EventType::RoomMember,
            content: serde_json::to_value(member::MemberEventContent {
                membership: member::MembershipState::Join,
-                displayname: db.users.displayname(&sender_user)?,
-                avatar_url: db.users.avatar_url(&sender_user)?,
+                displayname: db.users.displayname(sender_user)?,
+                avatar_url: db.users.avatar_url(sender_user)?,
                is_direct: Some(body.is_direct),
                third_party_invite: None,
+                blurhash: db.users.blurhash(sender_user)?,
+                reason: None,
            })
            .expect("event is valid, we just created it"),
            unsigned: None,
            state_key: Some(sender_user.to_string()),
            redacts: None,
        },
-        &sender_user,
+        sender_user,
        &room_id,
        &db,
+        &state_lock,
    )?;

    // 3. Power levels
-    let mut users = BTreeMap::new();
-    users.insert(sender_user.clone(), 100.into());
-    for invite_ in &body.invite {
-        users.insert(invite_.clone(), 100.into());
-    }
-
-    let power_levels_content = if let Some(power_levels) = &body.power_level_content_override {
-        serde_json::from_str(power_levels.json().get()).map_err(|_| {
-            Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.")
-        })?
-    } else {
-        serde_json::to_value(ruma::events::room::power_levels::PowerLevelsEventContent {
-            ban: 50.into(),
-            events: BTreeMap::new(),
-            events_default: 0.into(),
-            invite: 50.into(),
-            kick: 50.into(),
-            redact: 50.into(),
-            state_default: 50.into(),
-            users,
-            users_default: 0.into(),
-            notifications: ruma::events::room::power_levels::NotificationPowerLevels {
-                room: 50.into(),
-            },
-        })
-        .expect("event is valid, we just created it")
-    };
-    db.rooms.build_and_append_pdu(
-        PduBuilder {
-            event_type: EventType::RoomPowerLevels,
-            content: power_levels_content,
-            unsigned: None,
-            state_key: Some("".to_owned()),
-            redacts: None,
-        },
-        &sender_user,
-        &room_id,
-        &db,
-    )?;
-
-    // 4. Events set by preset

    // Figure out preset. We need it for preset specific events
    let preset = body
@@ -140,10 +145,78 @@ pub async fn create_room_route(
        .unwrap_or_else(|| match &body.visibility {
            room::Visibility::Private => create_room::RoomPreset::PrivateChat,
            room::Visibility::Public => create_room::RoomPreset::PublicChat,
-            room::Visibility::_Custom(s) => create_room::RoomPreset::_Custom(s.into()),
+            _ => create_room::RoomPreset::PrivateChat, // Room visibility should not be custom
        });

-    // 4.1 Join Rules
+    let mut users = BTreeMap::new();
+    users.insert(sender_user.clone(), 100.into());
+
+    if preset == create_room::RoomPreset::TrustedPrivateChat {
+        for invite_ in &body.invite {
+            users.insert(invite_.clone(), 100.into());
+        }
+    }
+
+    let mut power_levels_content =
+        serde_json::to_value(ruma::events::room::power_levels::PowerLevelsEventContent {
+            users,
+            ..Default::default()
+        })
+        .expect("event is valid, we just created it");
+
+    if let Some(power_level_content_override) = &body.power_level_content_override {
+        let json = serde_json::from_str::<serde_json::Map<String, serde_json::Value>>(
+            power_level_content_override.json().get(),
+        )
+        .map_err(|_| {
+            Error::BadRequest(ErrorKind::BadJson, "Invalid power_level_content_override.")
+        })?;
+
+        for (key, value) in json {
+            power_levels_content[key] = value;
+        }
+    }
+
+    db.rooms.build_and_append_pdu(
+        PduBuilder {
+            event_type: EventType::RoomPowerLevels,
+            content: power_levels_content,
+            unsigned: None,
+            state_key: Some("".to_owned()),
+            redacts: None,
+        },
+        sender_user,
+        &room_id,
+        &db,
+        &state_lock,
+    )?;
+
+    // 4. Canonical room alias
+    if let Some(room_alias_id) = &alias {
+        db.rooms.build_and_append_pdu(
+            PduBuilder {
+                event_type: EventType::RoomCanonicalAlias,
+                content: serde_json::to_value(
+                    ruma::events::room::canonical_alias::CanonicalAliasEventContent {
+                        alias: Some(room_alias_id.clone()),
+                        alt_aliases: vec![],
+                    },
+                )
+                .expect("We checked that alias earlier, it must be fine"),
+                unsigned: None,
+                state_key: Some("".to_owned()),
+                redacts: None,
+            },
+            sender_user,
+            &room_id,
+            &db,
+            &state_lock,
+        )?;
+    }
+
+    // 5. Events set by preset
+
+    // 5.1 Join Rules
    db.rooms.build_and_append_pdu(
        PduBuilder {
            event_type: EventType::RoomJoinRules,
@@ -162,12 +235,13 @@ pub async fn create_room_route(
            state_key: Some("".to_owned()),
            redacts: None,
        },
-        &sender_user,
+        sender_user,
        &room_id,
        &db,
+        &state_lock,
    )?;

-    // 4.2 History Visibility
+    // 5.2 History Visibility
    db.rooms.build_and_append_pdu(
        PduBuilder {
            event_type: EventType::RoomHistoryVisibility,
@@ -179,12 +253,13 @@ pub async fn create_room_route(
            state_key: Some("".to_owned()),
            redacts: None,
        },
-        &sender_user,
+        sender_user,
        &room_id,
        &db,
+        &state_lock,
    )?;

-    // 4.3 Guest Access
+    // 5.3 Guest Access
    db.rooms.build_and_append_pdu(
        PduBuilder {
            event_type: EventType::RoomGuestAccess,
@@ -204,17 +279,18 @@ pub async fn create_room_route(
            state_key: Some("".to_owned()),
            redacts: None,
        },
-        &sender_user,
+        sender_user,
        &room_id,
        &db,
+        &state_lock,
    )?;

-    // 5. Events listed in initial_state
+    // 6. Events listed in initial_state
    for event in &body.initial_state {
-        let pdu_builder = serde_json::from_str::<PduBuilder>(
-            &serde_json::to_string(&event).expect("AnyInitialStateEvent::to_string always works"),
-        )
-        .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event."))?;
+        let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| {
+            warn!("Invalid initial state event: {:?}", e);
+            Error::BadRequest(ErrorKind::InvalidParam, "Invalid initial state event.")
+        })?);

        // Silently skip encryption events if they are not allowed
        if pdu_builder.event_type == EventType::RoomEncryption && !db.globals.allow_encryption() {
@@ -222,27 +298,24 @@ pub async fn create_room_route(
        }

        db.rooms
-            .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db)?;
+            .build_and_append_pdu(pdu_builder, sender_user, &room_id, &db, &state_lock)?;
    }

-    // 6. Events implied by name and topic
+    // 7. Events implied by name and topic
    if let Some(name) = &body.name {
        db.rooms.build_and_append_pdu(
            PduBuilder {
                event_type: EventType::RoomName,
-                content: serde_json::to_value(
-                    name::NameEventContent::new(name.clone()).map_err(|_| {
-                        Error::BadRequest(ErrorKind::InvalidParam, "Name is invalid.")
-                    })?,
-                )
-                .expect("event is valid, we just created it"),
+                content: serde_json::to_value(name::NameEventContent::new(Some(name.clone())))
+                    .expect("event is valid, we just created it"),
                unsigned: None,
                state_key: Some("".to_owned()),
                redacts: None,
            },
-            &sender_user,
+            sender_user,
            &room_id,
            &db,
+            &state_lock,
        )?;
    }

@@ -258,33 +331,17 @@ pub async fn create_room_route(
                state_key: Some("".to_owned()),
                redacts: None,
            },
-            &sender_user,
+            sender_user,
            &room_id,
            &db,
+            &state_lock,
        )?;
    }

-    // 7. Events implied by invite (and TODO: invite_3pid)
-    for user in &body.invite {
-        db.rooms.build_and_append_pdu(
-            PduBuilder {
-                event_type: EventType::RoomMember,
-                content: serde_json::to_value(member::MemberEventContent {
-                    membership: member::MembershipState::Invite,
-                    displayname: db.users.displayname(&user)?,
-                    avatar_url: db.users.avatar_url(&user)?,
-                    is_direct: Some(body.is_direct),
-                    third_party_invite: None,
-                })
-                .expect("event is valid, we just created it"),
-                unsigned: None,
-                state_key: Some(user.to_string()),
-                redacts: None,
-            },
-            &sender_user,
-            &room_id,
-            &db,
-        )?;
+    // 8. Events implied by invite (and TODO: invite_3pid)
+    drop(state_lock);
+    for user_id in &body.invite {
+        let _ = invite_helper(sender_user, user_id, &room_id, &db, body.is_direct).await;
    }

    // Homeserver specific stuff
@@ -298,18 +355,23 @@ pub async fn create_room_route(

    info!("{} created a room", sender_user);

-    db.flush().await?;
+    db.flush()?;

    Ok(create_room::Response::new(room_id).into())
 }

+/// # `GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}`
+///
+/// Gets a single event.
+///
+/// - You have to currently be joined to the room (TODO: Respect history visibility)
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/rooms/<_>/event/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_room_event_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_room_event::Request<'_>>,
 ) -> ConduitResult<get_room_event::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -331,15 +393,57 @@ pub async fn get_room_event_route(
    .into())
 }

+/// # `GET /_matrix/client/r0/rooms/{roomId}/aliases`
+///
+/// Lists all aliases of the room.
+///
+/// - Only users joined to the room are allowed to call this TODO: Allow any user to call it if history_visibility is world readable
 #[cfg_attr(
    feature = "conduit_bin",
-    post("/_matrix/client/r0/rooms/<_room_id>/upgrade", data = "<body>")
+    get("/_matrix/client/r0/rooms/<_>/aliases", data = "<body>")
+)]
+#[tracing::instrument(skip(db, body))]
+pub async fn get_room_aliases_route(
+    db: DatabaseGuard,
+    body: Ruma<aliases::Request<'_>>,
+) -> ConduitResult<aliases::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+
+    if !db.rooms.is_joined(sender_user, &body.room_id)? {
+        return Err(Error::BadRequest(
+            ErrorKind::Forbidden,
+            "You don't have permission to view this room.",
+        ));
+    }
+
+    Ok(aliases::Response {
+        aliases: db
+            .rooms
+            .room_aliases(&body.room_id)
+            .filter_map(|a| a.ok())
+            .collect(),
+    }
+    .into())
+}
+
+/// # `GET /_matrix/client/r0/rooms/{roomId}/upgrade`
+///
+/// Upgrades the room.
+///
+/// - Creates a replacement room
+/// - Sends a tombstone event into the current room
+/// - Sender user joins the room
+/// - Transfers some state events
+/// - Moves local aliases
+/// - Modifies old room power levels to prevent users from speaking
+#[cfg_attr(
+    feature = "conduit_bin",
+    post("/_matrix/client/r0/rooms/<_>/upgrade", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn upgrade_room_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<upgrade_room::Request<'_>>,
-    _room_id: String,
 ) -> ConduitResult<upgrade_room::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

@@ -355,6 +459,18 @@ pub async fn upgrade_room_route(

    // Create a replacement room
    let replacement_room = RoomId::new(db.globals.server_name());
+    db.rooms
+        .get_or_create_shortroomid(&replacement_room, &db.globals)?;
+
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(body.room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;

    // Send a m.room.tombstone event to the old room to indicate that it is not intended to be used any further
    // Fail if the sender does not have the required permissions
@@ -373,15 +489,28 @@ pub async fn upgrade_room_route(
        sender_user,
        &body.room_id,
        &db,
+        &state_lock,
    )?;

+    // Change lock to replacement room
+    drop(state_lock);
+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(replacement_room.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
    // Get the old room federations status
    let federate = serde_json::from_value::<Raw<ruma::events::room::create::CreateEventContent>>(
        db.rooms
            .room_state_get(&body.room_id, &EventType::RoomCreate, "")?
            .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
-            .1
-            .content,
+            .content
+            .clone(),
    )
    .expect("Raw::from_value always works")
    .deserialize()
@@ -413,6 +542,7 @@ pub async fn upgrade_room_route(
        sender_user,
        &replacement_room,
        &db,
+        &state_lock,
    )?;

    // Join the new room
@@ -421,10 +551,12 @@ pub async fn upgrade_room_route(
            event_type: EventType::RoomMember,
            content: serde_json::to_value(member::MemberEventContent {
                membership: member::MembershipState::Join,
-                displayname: db.users.displayname(&sender_user)?,
-                avatar_url: db.users.avatar_url(&sender_user)?,
+                displayname: db.users.displayname(sender_user)?,
+                avatar_url: db.users.avatar_url(sender_user)?,
                is_direct: None,
                third_party_invite: None,
+                blurhash: db.users.blurhash(sender_user)?,
+                reason: None,
            })
            .expect("event is valid, we just created it"),
            unsigned: None,
@@ -434,6 +566,7 @@ pub async fn upgrade_room_route(
        sender_user,
        &replacement_room,
        &db,
+        &state_lock,
    )?;

    // Recommended transferable state events list from the specs
@@ -452,7 +585,7 @@ pub async fn upgrade_room_route(
    // Replicate transferable state events to the new room
    for event_type in transferable_state_events {
        let event_content = match db.rooms.room_state_get(&body.room_id, &event_type, "")? {
-            Some((_, v)) => v.content.clone(),
+            Some(v) => v.content.clone(),
            None => continue, // Skipping missing events.
        };

@@ -467,6 +600,7 @@ pub async fn upgrade_room_route(
            sender_user,
            &replacement_room,
            &db,
+            &state_lock,
        )?;
    }

@@ -482,8 +616,8 @@ pub async fn upgrade_room_route(
            db.rooms
                .room_state_get(&body.room_id, &EventType::RoomPowerLevels, "")?
                .ok_or_else(|| Error::bad_database("Found room without m.room.create event."))?
-                .1
-                .content,
+                .content
+                .clone(),
        )
        .expect("database contains invalid PDU")
        .deserialize()
@@ -510,9 +644,12 @@ pub async fn upgrade_room_route(
        sender_user,
        &body.room_id,
        &db,
+        &state_lock,
    )?;

-    db.flush().await?;
+    drop(state_lock);
+
+    db.flush()?;

    // Return the replacement room id
    Ok(upgrade_room::Response { replacement_room }.into())
@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
 use ruma::api::client::{error::ErrorKind, r0::search::search_events};

 #[cfg(feature = "conduit_bin")]
@@ -7,29 +6,49 @@ use rocket::post;
 use search_events::{EventContextResult, ResultCategories, ResultRoomEvents, SearchResult};
 use std::collections::BTreeMap;

+/// # `POST /_matrix/client/r0/search`
+///
+/// Searches rooms for messages.
+///
+/// - Only works if the user is currently joined to the room (TODO: Respect history visibility)
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/search", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn search_events_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<search_events::Request<'_>>,
 ) -> ConduitResult<search_events::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let search_criteria = body.search_categories.room_events.as_ref().unwrap();
-    let filter = search_criteria.filter.as_ref().unwrap();
+    let filter = search_criteria.filter.clone().unwrap_or_default();

-    let room_id = filter.rooms.as_ref().unwrap().first().unwrap();
+    let room_ids = filter.rooms.clone().unwrap_or_else(|| {
+        db.rooms
+            .rooms_joined(sender_user)
+            .filter_map(|r| r.ok())
+            .collect()
+    });

    let limit = filter.limit.map_or(10, |l| u64::from(l) as usize);

-    if !db.rooms.is_joined(sender_user, &room_id)? {
-        return Err(Error::BadRequest(
-            ErrorKind::Forbidden,
-            "You don't have permission to view this room.",
-        ));
+    let mut searches = Vec::new();
+
+    for room_id in room_ids {
+        if !db.rooms.is_joined(sender_user, &room_id)? {
+            return Err(Error::BadRequest(
+                ErrorKind::Forbidden,
+                "You don't have permission to view this room.",
+            ));
+        }
+
+        let search = db
+            .rooms
+            .search_pdus(&room_id, &search_criteria.search_term)?;
+
+        searches.push(search.0.peekable());
    }

    let skip = match body.next_batch.as_ref().map(|s| s.parse()) {
@@ -43,12 +62,20 @@ pub async fn search_events_route(
        None => 0, // Default to the start
    };

-    let search = db
-        .rooms
-        .search_pdus(&room_id, &search_criteria.search_term)?;
+    let mut results = Vec::new();
+    for _ in 0..skip + limit {
+        if let Some(s) = searches
+            .iter_mut()
+            .map(|s| (s.peek().cloned(), s))
+            .max_by_key(|(peek, _)| peek.clone())
+            .and_then(|(_, i)| i.next())
+        {
+            results.push(s);
+        }
+    }

-    let results = search
-        .0
+    let results = results
+        .iter()
        .map(|result| {
            Ok::<_, Error>(SearchResult {
                context: EventContextResult {
@@ -61,7 +88,7 @@ pub async fn search_events_route(
                rank: None,
                result: db
                    .rooms
-                    .get_pdu_from_id(&result)?
+                    .get_pdu_from_id(result)?
                    .map(|pdu| pdu.to_room_event()),
            })
        })
@@ -83,7 +110,11 @@ pub async fn search_events_route(
            next_batch,
            results,
            state: BTreeMap::new(), // TODO
-            highlights: search.1,
+            highlights: search_criteria
+                .search_term
+                .split_terminator(|c: char| !c.is_alphanumeric())
+                .map(str::to_lowercase)
+                .collect::<Vec<_>>(),
        },
    })
    .into())
@@ -1,14 +1,17 @@
-use super::{State, DEVICE_ID_LENGTH, TOKEN_LENGTH};
-use crate::{utils, ConduitResult, Database, Error, Ruma};
-use log::info;
+use super::{DEVICE_ID_LENGTH, TOKEN_LENGTH};
+use crate::{database::DatabaseGuard, utils, ConduitResult, Error, Ruma};
 use ruma::{
    api::client::{
        error::ErrorKind,
-        r0::session::{get_login_types, login, logout, logout_all},
+        r0::{
+            session::{get_login_types, login, logout, logout_all},
+            uiaa::IncomingUserIdentifier,
+        },
    },
    UserId,
 };
 use serde::Deserialize;
+use tracing::info;

 #[derive(Debug, Deserialize)]
 struct Claims {
@@ -21,21 +24,27 @@ use rocket::{get, post};

 /// # `GET /_matrix/client/r0/login`
 ///
-/// Get the homeserver's supported login types. One of these should be used as the `type` field
+/// Get the supported login types of this server. One of these should be used as the `type` field
 /// when logging in.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/login"))]
 #[tracing::instrument]
 pub async fn get_login_types_route() -> ConduitResult<get_login_types::Response> {
-    Ok(get_login_types::Response::new(vec![get_login_types::LoginType::Password]).into())
+    Ok(
+        get_login_types::Response::new(vec![get_login_types::LoginType::Password(
+            Default::default(),
+        )])
+        .into(),
+    )
 }

 /// # `POST /_matrix/client/r0/login`
 ///
 /// Authenticates the user and returns an access token it can use in subsequent requests.
 ///
-/// - The returned access token is associated with the user and device
-/// - Old access tokens of that device should be invalidated
-/// - If `device_id` is unknown, a new device will be created
+/// - The user needs to authenticate using their password (or if enabled using a json web token)
+/// - If `device_id` is known: invalidates old access token of that device
+/// - If `device_id` is unknown: creates a new device
+/// - Returns access token that is associated with the user and device
 ///
 /// Note: You can use [`GET /_matrix/client/r0/login`](fn.get_supported_versions_route.html) to see
 /// supported login types.
@@ -45,14 +54,17 @@ pub async fn get_login_types_route() -> ConduitResult<get_login_types::Response>
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn login_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<login::Request<'_>>,
 ) -> ConduitResult<login::Response> {
    // Validate login method
    // TODO: Other login methods
    let user_id = match &body.login_info {
-        login::IncomingLoginInfo::Password { password } => {
-            let username = if let login::IncomingUserInfo::MatrixId(matrix_id) = &body.user {
+        login::IncomingLoginInfo::Password {
+            identifier,
+            password,
+        } => {
+            let username = if let IncomingUserIdentifier::MatrixId(matrix_id) = identifier {
                matrix_id
            } else {
                return Err(Error::BadRequest(ErrorKind::Forbidden, "Bad login type."));
@@ -88,8 +100,8 @@ pub async fn login_route(
        login::IncomingLoginInfo::Token { token } => {
            if let Some(jwt_decoding_key) = db.globals.jwt_decoding_key() {
                let token = jsonwebtoken::decode::<Claims>(
-                    &token,
-                    &jwt_decoding_key,
+                    token,
+                    jwt_decoding_key,
                    &jsonwebtoken::Validation::default(),
                )
                .map_err(|_| Error::BadRequest(ErrorKind::InvalidUsername, "Token is invalid."))?;
@@ -135,7 +147,7 @@ pub async fn login_route(

    info!("{} logged in", user_id);

-    db.flush().await?;
+    db.flush()?;

    Ok(login::Response {
        user_id,
@@ -151,23 +163,25 @@ pub async fn login_route(
 ///
 /// Log out the current device.
 ///
-/// - Invalidates the access token
-/// - Deletes the device and most of it's data (to-device events, last seen, etc.)
+/// - Invalidates access token
+/// - Deletes device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets to-device events
+/// - Triggers device list updates
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/logout", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn logout_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<logout::Request>,
 ) -> ConduitResult<logout::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
    let sender_device = body.sender_device.as_ref().expect("user is authenticated");

-    db.users.remove_device(&sender_user, sender_device)?;
+    db.users.remove_device(sender_user, sender_device)?;

-    db.flush().await?;
+    db.flush()?;

    Ok(logout::Response::new().into())
 }
@@ -177,7 +191,9 @@ pub async fn logout_route(
 /// Log out all devices of this user.
 ///
 /// - Invalidates all access tokens
-/// - Deletes devices and most of their data (to-device events, last seen, etc.)
+/// - Deletes all device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets all to-device events
+/// - Triggers device list updates
 ///
 /// Note: This is equivalent to calling [`GET /_matrix/client/r0/logout`](fn.logout_route.html)
 /// from each device of this user.
@@ -187,18 +203,16 @@ pub async fn logout_route(
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn logout_all_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<logout_all::Request>,
 ) -> ConduitResult<logout_all::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    for device_id in db.users.all_device_ids(sender_user) {
-        if let Ok(device_id) = device_id {
-            db.users.remove_device(&sender_user, &device_id)?;
-        }
+    for device_id in db.users.all_device_ids(sender_user).flatten() {
+        db.users.remove_device(sender_user, &device_id)?;
    }

-    db.flush().await?;
+    db.flush()?;

    Ok(logout_all::Response::new().into())
 }
@@ -1,106 +1,105 @@
-use super::State;
-use crate::{pdu::PduBuilder, ConduitResult, Database, Error, Result, Ruma};
+use std::sync::Arc;
+
+use crate::{
+    database::DatabaseGuard, pdu::PduBuilder, ConduitResult, Database, Error, Result, Ruma,
+};
 use ruma::{
    api::client::{
        error::ErrorKind,
-        r0::state::{
-            get_state_events, get_state_events_for_empty_key, get_state_events_for_key,
-            send_state_event_for_empty_key, send_state_event_for_key,
-        },
+        r0::state::{get_state_events, get_state_events_for_key, send_state_event},
    },
    events::{
-        room::history_visibility::{HistoryVisibility, HistoryVisibilityEventContent},
-        AnyStateEventContent, EventContent, EventType,
+        room::{
+            canonical_alias::CanonicalAliasEventContent,
+            history_visibility::{HistoryVisibility, HistoryVisibilityEventContent},
+        },
+        AnyStateEventContent, EventType,
    },
+    serde::Raw,
    EventId, RoomId, UserId,
 };

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};

+/// # `PUT /_matrix/client/r0/rooms/{roomId}/state/{eventType}/{stateKey}`
+///
+/// Sends a state event into the room.
+///
+/// - The only requirement for the content is that it has to be valid json
+/// - Tries to send the event into the room, auth rules will determine if it is allowed
+/// - If event is new canonical_alias: Rejects if alias is incorrect
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn send_state_event_for_key_route(
-    db: State<'_, Database>,
-    body: Ruma<send_state_event_for_key::Request<'_>>,
-) -> ConduitResult<send_state_event_for_key::Response> {
+    db: DatabaseGuard,
+    body: Ruma<send_state_event::Request<'_>>,
+) -> ConduitResult<send_state_event::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

-    let content = serde_json::from_str::<serde_json::Value>(
-        body.json_body
-            .as_ref()
-            .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?
-            .get(),
-    )
-    .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?;
-
    let event_id = send_state_event_for_key_helper(
        &db,
        sender_user,
-        &body.content,
-        content,
        &body.room_id,
-        Some(body.state_key.to_owned()),
+        EventType::from(&body.event_type),
+        &body.body.body, // Yes, I hate it too
+        body.state_key.to_owned(),
    )
    .await?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(send_state_event_for_key::Response { event_id }.into())
+    Ok(send_state_event::Response { event_id }.into())
 }

+/// # `PUT /_matrix/client/r0/rooms/{roomId}/state/{eventType}`
+///
+/// Sends a state event into the room.
+///
+/// - The only requirement for the content is that it has to be valid json
+/// - Tries to send the event into the room, auth rules will determine if it is allowed
+/// - If event is new canonical_alias: Rejects if alias is incorrect
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/rooms/<_>/state/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn send_state_event_for_empty_key_route(
-    db: State<'_, Database>,
-    body: Ruma<send_state_event_for_empty_key::Request<'_>>,
-) -> ConduitResult<send_state_event_for_empty_key::Response> {
-    // This just calls send_state_event_for_key_route
-    let Ruma {
-        body,
-        sender_user,
-        json_body,
-        ..
-    } = body;
-
-    let json = serde_json::from_str::<serde_json::Value>(
-        json_body
-            .as_ref()
-            .ok_or(Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?
-            .get(),
-    )
-    .map_err(|_| Error::BadRequest(ErrorKind::BadJson, "Invalid JSON body."))?;
+    db: DatabaseGuard,
+    body: Ruma<send_state_event::Request<'_>>,
+) -> ConduitResult<send_state_event::Response> {
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    let event_id = send_state_event_for_key_helper(
        &db,
-        sender_user
-            .as_ref()
-            .expect("no user for send state empty key rout"),
-        &body.content,
-        json,
+        sender_user,
        &body.room_id,
-        Some("".into()),
+        EventType::from(&body.event_type),
+        &body.body.body,
+        body.state_key.to_owned(),
    )
    .await?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(send_state_event_for_empty_key::Response { event_id }.into())
+    Ok(send_state_event::Response { event_id }.into())
 }

+/// # `GET /_matrix/client/r0/rooms/{roomid}/state`
+///
+/// Get all state events for a room.
+///
+/// - If not joined: Only works if current room history visibility is world readable
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/rooms/<_>/state", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_state_events_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_state_events::Request<'_>>,
 ) -> ConduitResult<get_state_events::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -112,8 +111,8 @@ pub async fn get_state_events_route(
        && !matches!(
            db.rooms
                .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
-                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
+                .map(|event| {
+                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
                        .map_err(|_| {
                            Error::bad_database(
                                "Invalid room history visibility event in database.",
@@ -141,13 +140,18 @@ pub async fn get_state_events_route(
    .into())
 }

+/// # `GET /_matrix/client/r0/rooms/{roomid}/state/{eventType}/{stateKey}`
+///
+/// Get single state event of a room.
+///
+/// - If not joined: Only works if current room history visibility is world readable
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_state_events_for_key_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_state_events_for_key::Request<'_>>,
 ) -> ConduitResult<get_state_events_for_key::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -159,8 +163,8 @@ pub async fn get_state_events_for_key_route(
        && !matches!(
            db.rooms
                .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
-                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
+                .map(|event| {
+                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
                        .map_err(|_| {
                            Error::bad_database(
                                "Invalid room history visibility event in database.",
@@ -183,25 +187,29 @@ pub async fn get_state_events_for_key_route(
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "State event not found.",
-        ))?
-        .1;
+        ))?;

    Ok(get_state_events_for_key::Response {
-        content: serde_json::value::to_raw_value(&event.content)
+        content: serde_json::from_value(event.content.clone())
            .map_err(|_| Error::bad_database("Invalid event content in database"))?,
    }
    .into())
 }

+/// # `GET /_matrix/client/r0/rooms/{roomid}/state/{eventType}`
+///
+/// Get single state event of a room.
+///
+/// - If not joined: Only works if current room history visibility is world readable
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/rooms/<_>/state/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_state_events_for_empty_key_route(
-    db: State<'_, Database>,
-    body: Ruma<get_state_events_for_empty_key::Request<'_>>,
-) -> ConduitResult<get_state_events_for_empty_key::Response> {
+    db: DatabaseGuard,
+    body: Ruma<get_state_events_for_key::Request<'_>>,
+) -> ConduitResult<get_state_events_for_key::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    #[allow(clippy::blocks_in_if_conditions)]
@@ -211,8 +219,8 @@ pub async fn get_state_events_for_empty_key_route(
        && !matches!(
            db.rooms
                .room_state_get(&body.room_id, &EventType::RoomHistoryVisibility, "")?
-                .map(|(_, event)| {
-                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content)
+                .map(|event| {
+                    serde_json::from_value::<HistoryVisibilityEventContent>(event.content.clone())
                        .map_err(|_| {
                            Error::bad_database(
                                "Invalid room history visibility event in database.",
@@ -235,30 +243,33 @@ pub async fn get_state_events_for_empty_key_route(
        .ok_or(Error::BadRequest(
            ErrorKind::NotFound,
            "State event not found.",
-        ))?
-        .1;
+        ))?;

-    Ok(get_state_events_for_empty_key::Response {
-        content: serde_json::value::to_raw_value(&event.content)
+    Ok(get_state_events_for_key::Response {
+        content: serde_json::from_value(event.content.clone())
            .map_err(|_| Error::bad_database("Invalid event content in database"))?,
    }
    .into())
 }

-pub async fn send_state_event_for_key_helper(
+async fn send_state_event_for_key_helper(
    db: &Database,
    sender: &UserId,
-    content: &AnyStateEventContent,
-    json: serde_json::Value,
    room_id: &RoomId,
-    state_key: Option<String>,
+    event_type: EventType,
+    json: &Raw<AnyStateEventContent>,
+    state_key: String,
 ) -> Result<EventId> {
    let sender_user = sender;

-    if let AnyStateEventContent::RoomCanonicalAlias(canonical_alias) = content {
+    // TODO: Review this check, error if event is unparsable, use event type, allow alias if it
+    // previously existed
+    if let Ok(canonical_alias) =
+        serde_json::from_str::<CanonicalAliasEventContent>(json.json().get())
+    {
        let mut aliases = canonical_alias.alt_aliases.clone();

-        if let Some(alias) = canonical_alias.alias.clone() {
+        if let Some(alias) = canonical_alias.alias {
            aliases.push(alias);
        }

@@ -279,17 +290,28 @@ pub async fn send_state_event_for_key_helper(
        }
    }

+    let mutex_state = Arc::clone(
+        db.globals
+            .roomid_mutex_state
+            .write()
+            .unwrap()
+            .entry(room_id.clone())
+            .or_default(),
+    );
+    let state_lock = mutex_state.lock().await;
+
    let event_id = db.rooms.build_and_append_pdu(
        PduBuilder {
-            event_type: content.event_type().into(),
-            content: json,
+            event_type,
+            content: serde_json::from_str(json.json().get()).expect("content is valid json"),
            unsigned: None,
-            state_key,
+            state_key: Some(state_key),
            redacts: None,
        },
-        &sender_user,
-        &room_id,
-        &db,
+        sender_user,
+        room_id,
+        db,
+        &state_lock,
    )?;

    Ok(event_id)
@@ -1,5 +1,4 @@
-use super::State;
-use crate::{ConduitResult, Database, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Ruma};
 use ruma::{
    api::client::r0::tag::{create_tag, delete_tag, get_tags},
    events::EventType,
@@ -9,13 +8,18 @@ use std::collections::BTreeMap;
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, put};

+/// # `PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags/{tag}`
+///
+/// Adds a tag to the room.
+///
+/// - Inserts the tag into the tag event of the room account data.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/user/<_>/rooms/<_>/tags/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn update_tag_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<create_tag::Request<'_>>,
 ) -> ConduitResult<create_tag::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -31,7 +35,7 @@ pub async fn update_tag_route(
    tags_event
        .content
        .tags
-        .insert(body.tag.to_string(), body.tag_info.clone());
+        .insert(body.tag.clone().into(), body.tag_info.clone());

    db.account_data.update(
        Some(&body.room_id),
@@ -41,18 +45,23 @@ pub async fn update_tag_route(
        &db.globals,
    )?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(create_tag::Response.into())
+    Ok(create_tag::Response {}.into())
 }

+/// # `DELETE /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags/{tag}`
+///
+/// Deletes a tag from the room.
+///
+/// - Removes the tag from the tag event of the room account data.
 #[cfg_attr(
    feature = "conduit_bin",
    delete("/_matrix/client/r0/user/<_>/rooms/<_>/tags/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn delete_tag_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<delete_tag::Request<'_>>,
 ) -> ConduitResult<delete_tag::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -65,7 +74,7 @@ pub async fn delete_tag_route(
                tags: BTreeMap::new(),
            },
        });
-    tags_event.content.tags.remove(&body.tag);
+    tags_event.content.tags.remove(&body.tag.clone().into());

    db.account_data.update(
        Some(&body.room_id),
@@ -75,18 +84,23 @@ pub async fn delete_tag_route(
        &db.globals,
    )?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(delete_tag::Response.into())
+    Ok(delete_tag::Response {}.into())
 }

+/// # `GET /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags`
+///
+/// Returns tags on the room.
+///
+/// - Gets the tag event of the room account data.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/user/<_>/rooms/<_>/tags", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn get_tags_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<get_tags::Request<'_>>,
 ) -> ConduitResult<get_tags::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
@@ -1,18 +1,20 @@
 use crate::ConduitResult;
 use ruma::api::client::r0::thirdparty::get_protocols;

-use log::warn;
 #[cfg(feature = "conduit_bin")]
 use rocket::get;
 use std::collections::BTreeMap;

+/// # `GET /_matrix/client/r0/thirdparty/protocols`
+///
+/// TODO: Fetches all metadata about protocols supported by the homeserver.
 #[cfg_attr(
    feature = "conduit_bin",
    get("/_matrix/client/r0/thirdparty/protocols")
 )]
 #[tracing::instrument]
 pub async fn get_protocols_route() -> ConduitResult<get_protocols::Response> {
-    warn!("TODO: get_protocols_route");
+    // TODO
    Ok(get_protocols::Response {
        protocols: BTreeMap::new(),
    }
@@ -1,26 +1,36 @@
-use super::State;
-use crate::{ConduitResult, Database, Error, Ruma};
-use ruma::api::client::{
-    error::ErrorKind,
-    r0::to_device::{self, send_event_to_device},
+use std::collections::BTreeMap;
+
+use crate::{database::DatabaseGuard, ConduitResult, Error, Ruma};
+use ruma::{
+    api::{
+        client::{error::ErrorKind, r0::to_device::send_event_to_device},
+        federation::{self, transactions::edu::DirectDeviceContent},
+    },
+    events::EventType,
+    to_device::DeviceIdOrAllDevices,
 };

 #[cfg(feature = "conduit_bin")]
 use rocket::put;

+/// # `PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}`
+///
+/// Send a to-device event to a set of client devices.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/sendToDevice/<_>/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn send_event_to_device_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<send_event_to_device::Request<'_>>,
 ) -> ConduitResult<send_event_to_device::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
    let sender_device = body.sender_device.as_deref();

+    // TODO: uncomment when https://github.com/vector-im/element-android/issues/3589 is solved
    // Check if this is a new transaction id
+    /*
    if db
        .transaction_ids
        .existing_txnid(sender_user, sender_device, &body.txn_id)?
@@ -28,31 +38,53 @@ pub async fn send_event_to_device_route(
    {
        return Ok(send_event_to_device::Response.into());
    }
+    */

    for (target_user_id, map) in &body.messages {
        for (target_device_id_maybe, event) in map {
-            match target_device_id_maybe {
-                to_device::DeviceIdOrAllDevices::DeviceId(target_device_id) => {
-                    db.users.add_to_device_event(
-                        sender_user,
-                        &target_user_id,
-                        &target_device_id,
-                        &body.event_type,
-                        serde_json::from_str(event.get()).map_err(|_| {
-                            Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid")
-                        })?,
-                        &db.globals,
-                    )?
-                }
+            if target_user_id.server_name() != db.globals.server_name() {
+                let mut map = BTreeMap::new();
+                map.insert(target_device_id_maybe.clone(), event.clone());
+                let mut messages = BTreeMap::new();
+                messages.insert(target_user_id.clone(), map);

-                to_device::DeviceIdOrAllDevices::AllDevices => {
-                    for target_device_id in db.users.all_device_ids(&target_user_id) {
+                db.sending.send_reliable_edu(
+                    target_user_id.server_name(),
+                    serde_json::to_vec(&federation::transactions::edu::Edu::DirectToDevice(
+                        DirectDeviceContent {
+                            sender: sender_user.clone(),
+                            ev_type: EventType::from(&body.event_type),
+                            message_id: body.txn_id.clone(),
+                            messages,
+                        },
+                    ))
+                    .expect("DirectToDevice EDU can be serialized"),
+                    db.globals.next_count()?,
+                )?;
+
+                continue;
+            }
+
+            match target_device_id_maybe {
+                DeviceIdOrAllDevices::DeviceId(target_device_id) => db.users.add_to_device_event(
+                    sender_user,
+                    target_user_id,
+                    target_device_id,
+                    &body.event_type,
+                    event.deserialize_as().map_err(|_| {
+                        Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid")
+                    })?,
+                    &db.globals,
+                )?,
+
+                DeviceIdOrAllDevices::AllDevices => {
+                    for target_device_id in db.users.all_device_ids(target_user_id) {
                        db.users.add_to_device_event(
                            sender_user,
-                            &target_user_id,
+                            target_user_id,
                            &target_device_id?,
                            &body.event_type,
-                            serde_json::from_str(event.get()).map_err(|_| {
+                            event.deserialize_as().map_err(|_| {
                                Error::BadRequest(ErrorKind::InvalidParam, "Event is invalid")
                            })?,
                            &db.globals,
@@ -67,7 +99,7 @@ pub async fn send_event_to_device_route(
    db.transaction_ids
        .add_txnid(sender_user, sender_device, &body.txn_id, &[])?;

-    db.flush().await?;
+    db.flush()?;

-    Ok(send_event_to_device::Response.into())
+    Ok(send_event_to_device::Response {}.into())
 }
@@ -1,25 +1,27 @@
-use super::State;
-use crate::{utils, ConduitResult, Database, Ruma};
+use crate::{database::DatabaseGuard, utils, ConduitResult, Ruma};
 use create_typing_event::Typing;
 use ruma::api::client::r0::typing::create_typing_event;

 #[cfg(feature = "conduit_bin")]
 use rocket::put;

+/// # `PUT /_matrix/client/r0/rooms/{roomId}/typing/{userId}`
+///
+/// Sets the typing state of the sender user.
 #[cfg_attr(
    feature = "conduit_bin",
    put("/_matrix/client/r0/rooms/<_>/typing/<_>", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub fn create_typing_event_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<create_typing_event::Request<'_>>,
 ) -> ConduitResult<create_typing_event::Response> {
    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    if let Typing::Yes(duration) = body.state {
        db.rooms.edus.typing_add(
-            &sender_user,
+            sender_user,
            &body.room_id,
            duration.as_millis() as u64 + utils::millis_since_unix_epoch(),
            &db.globals,
@@ -27,8 +29,8 @@ pub fn create_typing_event_route(
    } else {
        db.rooms
            .edus
-            .typing_remove(&sender_user, &body.room_id, &db.globals)?;
+            .typing_remove(sender_user, &body.room_id, &db.globals)?;
    }

-    Ok(create_typing_event::Response.into())
+    Ok(create_typing_event::Response {}.into())
 }
@@ -10,7 +10,7 @@ use rocket::get;
 ///
 /// - Versions take the form MAJOR.MINOR.PATCH
 /// - Only the latest PATCH release will be reported for each MAJOR.MINOR value
-/// - Unstable features should be namespaced and may include version information in their name
+/// - Unstable features are namespaced and may include version information in their name
 ///
 /// Note: Unstable features are used while developing new features. Clients should avoid using
 /// unstable features in their stable releases
@@ -1,17 +1,21 @@
-use super::State;
-use crate::{ConduitResult, Database, Ruma};
+use crate::{database::DatabaseGuard, ConduitResult, Ruma};
 use ruma::api::client::r0::user_directory::search_users;

 #[cfg(feature = "conduit_bin")]
 use rocket::post;

+/// # `POST /_matrix/client/r0/user_directory/search`
+///
+/// Searches all known users for a match.
+///
+/// - TODO: Hide users that are not in any public rooms?
 #[cfg_attr(
    feature = "conduit_bin",
    post("/_matrix/client/r0/user_directory/search", data = "<body>")
 )]
 #[tracing::instrument(skip(db, body))]
 pub async fn search_users_route(
-    db: State<'_, Database>,
+    db: DatabaseGuard,
    body: Ruma<search_users::Request<'_>>,
 ) -> ConduitResult<search_users::Response> {
    let limit = u64::from(body.limit) as usize;
@@ -19,9 +23,6 @@ pub async fn search_users_route(
    let mut users = db.users.iter().filter_map(|user_id| {
        // Filter out buggy users (they should not exist, but you never know...)
        let user_id = user_id.ok()?;
-        if db.users.is_deactivated(&user_id).ok()? {
-            return None;
-        }

        let user = search_users::User {
            user_id: user_id.clone(),
@@ -29,13 +30,22 @@ pub async fn search_users_route(
            avatar_url: db.users.avatar_url(&user_id).ok()?,
        };

-        if !user.user_id.to_string().contains(&body.search_term)
-            && user
-                .display_name
-                .as_ref()
-                .filter(|name| name.contains(&body.search_term))
-                .is_none()
-        {
+        let user_id_matches = user
+            .user_id
+            .to_string()
+            .to_lowercase()
+            .contains(&body.search_term.to_lowercase());
+
+        let user_displayname_matches = user
+            .display_name
+            .as_ref()
+            .filter(|name| {
+                name.to_lowercase()
+                    .contains(&body.search_term.to_lowercase())
+            })
+            .is_some();
+
+        if !user_id_matches && !user_displayname_matches {
            return None;
        }

@@ -5,6 +5,9 @@ use std::time::Duration;
 #[cfg(feature = "conduit_bin")]
 use rocket::get;

+/// # `GET /_matrix/client/r0/voip/turnServer`
+///
+/// TODO: Returns information about the recommended turn server.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/voip/turnServer"))]
 #[tracing::instrument]
 pub async fn turn_server_route() -> ConduitResult<get_turn_server_info::Response> {
@@ -0,0 +1,54 @@
+use super::Config;
+use crate::Result;
+
+use std::{future::Future, pin::Pin, sync::Arc};
+
+#[cfg(feature = "sled")]
+pub mod sled;
+
+#[cfg(feature = "sqlite")]
+pub mod sqlite;
+
+#[cfg(feature = "heed")]
+pub mod heed;
+
+pub trait DatabaseEngine: Sized {
+    fn open(config: &Config) -> Result<Arc<Self>>;
+    fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>>;
+    fn flush(self: &Arc<Self>) -> Result<()>;
+}
+
+pub trait Tree: Send + Sync {
+    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>>;
+
+    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()>;
+    fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()>;
+
+    fn remove(&self, key: &[u8]) -> Result<()>;
+
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
+
+    fn iter_from<'a>(
+        &'a self,
+        from: &[u8],
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
+
+    fn increment(&self, key: &[u8]) -> Result<Vec<u8>>;
+    fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()>;
+
+    fn scan_prefix<'a>(
+        &'a self,
+        prefix: Vec<u8>,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
+
+    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>>;
+
+    fn clear(&self) -> Result<()> {
+        for (key, _) in self.iter() {
+            self.remove(&key)?;
+        }
+
+        Ok(())
+    }
+}
@@ -0,0 +1,240 @@
+use super::super::Config;
+use crossbeam::channel::{bounded, Sender as ChannelSender};
+use threadpool::ThreadPool;
+
+use crate::{Error, Result};
+use std::{
+    collections::HashMap,
+    future::Future,
+    pin::Pin,
+    sync::{Arc, Mutex, RwLock},
+};
+use tokio::sync::oneshot::Sender;
+
+use super::{DatabaseEngine, Tree};
+
+type TupleOfBytes = (Vec<u8>, Vec<u8>);
+
+pub struct Engine {
+    env: heed::Env,
+    iter_pool: Mutex<ThreadPool>,
+}
+
+pub struct EngineTree {
+    engine: Arc<Engine>,
+    tree: Arc<heed::UntypedDatabase>,
+    watchers: RwLock<HashMap<Vec<u8>, Vec<Sender<()>>>>,
+}
+
+fn convert_error(error: heed::Error) -> Error {
+    Error::HeedError {
+        error: error.to_string(),
+    }
+}
+
+impl DatabaseEngine for Engine {
+    fn open(config: &Config) -> Result<Arc<Self>> {
+        let mut env_builder = heed::EnvOpenOptions::new();
+        env_builder.map_size(1024 * 1024 * 1024 * 1024); // 1 Terabyte
+        env_builder.max_readers(126);
+        env_builder.max_dbs(128);
+        unsafe {
+            env_builder.flag(heed::flags::Flags::MdbWriteMap);
+            env_builder.flag(heed::flags::Flags::MdbMapAsync);
+        }
+
+        Ok(Arc::new(Engine {
+            env: env_builder
+                .open(&config.database_path)
+                .map_err(convert_error)?,
+            iter_pool: Mutex::new(ThreadPool::new(10)),
+        }))
+    }
+
+    fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> {
+        // Creates the db if it doesn't exist already
+        Ok(Arc::new(EngineTree {
+            engine: Arc::clone(self),
+            tree: Arc::new(
+                self.env
+                    .create_database(Some(name))
+                    .map_err(convert_error)?,
+            ),
+            watchers: RwLock::new(HashMap::new()),
+        }))
+    }
+
+    fn flush(self: &Arc<Self>) -> Result<()> {
+        self.env.force_sync().map_err(convert_error)?;
+        Ok(())
+    }
+}
+
+impl EngineTree {
+    #[tracing::instrument(skip(self, tree, from, backwards))]
+    fn iter_from_thread(
+        &self,
+        tree: Arc<heed::UntypedDatabase>,
+        from: Vec<u8>,
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = TupleOfBytes> + Send + Sync> {
+        let (s, r) = bounded::<TupleOfBytes>(100);
+        let engine = Arc::clone(&self.engine);
+
+        let lock = self.engine.iter_pool.lock().await;
+        if lock.active_count() < lock.max_count() {
+            lock.execute(move || {
+                iter_from_thread_work(tree, &engine.env.read_txn().unwrap(), from, backwards, &s);
+            });
+        } else {
+            std::thread::spawn(move || {
+                iter_from_thread_work(tree, &engine.env.read_txn().unwrap(), from, backwards, &s);
+            });
+        }
+
+        Box::new(r.into_iter())
+    }
+}
+
+#[tracing::instrument(skip(tree, txn, from, backwards))]
+fn iter_from_thread_work(
+    tree: Arc<heed::UntypedDatabase>,
+    txn: &heed::RoTxn<'_>,
+    from: Vec<u8>,
+    backwards: bool,
+    s: &ChannelSender<(Vec<u8>, Vec<u8>)>,
+) {
+    if backwards {
+        for (k, v) in tree.rev_range(txn, ..=&*from).unwrap().map(|r| r.unwrap()) {
+            if s.send((k.to_vec(), v.to_vec())).is_err() {
+                return;
+            }
+        }
+    } else {
+        if from.is_empty() {
+            for (k, v) in tree.iter(txn).unwrap().map(|r| r.unwrap()) {
+                if s.send((k.to_vec(), v.to_vec())).is_err() {
+                    return;
+                }
+            }
+        } else {
+            for (k, v) in tree.range(txn, &*from..).unwrap().map(|r| r.unwrap()) {
+                if s.send((k.to_vec(), v.to_vec())).is_err() {
+                    return;
+                }
+            }
+        }
+    }
+}
+
+impl Tree for EngineTree {
+    #[tracing::instrument(skip(self, key))]
+    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        let txn = self.engine.env.read_txn().map_err(convert_error)?;
+        Ok(self
+            .tree
+            .get(&txn, &key)
+            .map_err(convert_error)?
+            .map(|s| s.to_vec()))
+    }
+
+    #[tracing::instrument(skip(self, key, value))]
+    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
+        let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
+        self.tree
+            .put(&mut txn, &key, &value)
+            .map_err(convert_error)?;
+        txn.commit().map_err(convert_error)?;
+
+        let watchers = self.watchers.read().unwrap();
+        let mut triggered = Vec::new();
+
+        for length in 0..=key.len() {
+            if watchers.contains_key(&key[..length]) {
+                triggered.push(&key[..length]);
+            }
+        }
+
+        drop(watchers);
+
+        if !triggered.is_empty() {
+            let mut watchers = self.watchers.write().unwrap();
+            for prefix in triggered {
+                if let Some(txs) = watchers.remove(prefix) {
+                    for tx in txs {
+                        let _ = tx.send(());
+                    }
+                }
+            }
+        };
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, key))]
+    fn remove(&self, key: &[u8]) -> Result<()> {
+        let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
+        self.tree.delete(&mut txn, &key).map_err(convert_error)?;
+        txn.commit().map_err(convert_error)?;
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> {
+        self.iter_from(&[], false)
+    }
+
+    #[tracing::instrument(skip(self, from, backwards))]
+    fn iter_from(
+        &self,
+        from: &[u8],
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send> {
+        self.iter_from_thread(Arc::clone(&self.tree), from.to_vec(), backwards)
+    }
+
+    #[tracing::instrument(skip(self, key))]
+    fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
+        let mut txn = self.engine.env.write_txn().map_err(convert_error)?;
+
+        let old = self.tree.get(&txn, &key).map_err(convert_error)?;
+        let new =
+            crate::utils::increment(old.as_deref()).expect("utils::increment always returns Some");
+
+        self.tree
+            .put(&mut txn, &key, &&*new)
+            .map_err(convert_error)?;
+
+        txn.commit().map_err(convert_error)?;
+
+        Ok(new)
+    }
+
+    #[tracing::instrument(skip(self, prefix))]
+    fn scan_prefix<'a>(
+        &'a self,
+        prefix: Vec<u8>,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + Send + 'a> {
+        Box::new(
+            self.iter_from(&prefix, false)
+                .take_while(move |(key, _)| key.starts_with(&prefix)),
+        )
+    }
+
+    #[tracing::instrument(skip(self, prefix))]
+    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
+        let (tx, rx) = tokio::sync::oneshot::channel();
+
+        self.watchers
+            .write()
+            .unwrap()
+            .entry(prefix.to_vec())
+            .or_default()
+            .push(tx);
+
+        Box::pin(async move {
+            // Tx is never destroyed
+            rx.await.unwrap();
+        })
+    }
+}
@@ -0,0 +1,128 @@
+use super::super::Config;
+use crate::{utils, Result};
+use std::{future::Future, pin::Pin, sync::Arc};
+use tracing::warn;
+
+use super::{DatabaseEngine, Tree};
+
+pub struct Engine(sled::Db);
+
+pub struct SledEngineTree(sled::Tree);
+
+impl DatabaseEngine for Engine {
+    fn open(config: &Config) -> Result<Arc<Self>> {
+        Ok(Arc::new(Engine(
+            sled::Config::default()
+                .path(&config.database_path)
+                .cache_capacity((config.db_cache_capacity_mb * 1024.0 * 1024.0) as u64)
+                .use_compression(true)
+                .open()?,
+        )))
+    }
+
+    fn open_tree(self: &Arc<Self>, name: &'static str) -> Result<Arc<dyn Tree>> {
+        Ok(Arc::new(SledEngineTree(self.0.open_tree(name)?)))
+    }
+
+    fn flush(self: &Arc<Self>) -> Result<()> {
+        Ok(()) // noop
+    }
+}
+
+impl Tree for SledEngineTree {
+    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        Ok(self.0.get(key)?.map(|v| v.to_vec()))
+    }
+
+    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
+        self.0.insert(key, value)?;
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, iter))]
+    fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
+        for (key, value) in iter {
+            self.0.insert(key, value)?;
+        }
+
+        Ok(())
+    }
+
+    fn remove(&self, key: &[u8]) -> Result<()> {
+        self.0.remove(key)?;
+        Ok(())
+    }
+
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
+        Box::new(
+            self.0
+                .iter()
+                .filter_map(|r| {
+                    if let Err(e) = &r {
+                        warn!("Error: {}", e);
+                    }
+                    r.ok()
+                })
+                .map(|(k, v)| (k.to_vec().into(), v.to_vec().into())),
+        )
+    }
+
+    fn iter_from(
+        &self,
+        from: &[u8],
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)>> {
+        let iter = if backwards {
+            self.0.range(..=from)
+        } else {
+            self.0.range(from..)
+        };
+
+        let iter = iter
+            .filter_map(|r| {
+                if let Err(e) = &r {
+                    warn!("Error: {}", e);
+                }
+                r.ok()
+            })
+            .map(|(k, v)| (k.to_vec().into(), v.to_vec().into()));
+
+        if backwards {
+            Box::new(iter.rev())
+        } else {
+            Box::new(iter)
+        }
+    }
+
+    fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
+        Ok(self
+            .0
+            .update_and_fetch(key, utils::increment)
+            .map(|o| o.expect("increment always sets a value").to_vec())?)
+    }
+
+    fn scan_prefix<'a>(
+        &'a self,
+        prefix: Vec<u8>,
+    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a> {
+        let iter = self
+            .0
+            .scan_prefix(prefix)
+            .filter_map(|r| {
+                if let Err(e) = &r {
+                    warn!("Error: {}", e);
+                }
+                r.ok()
+            })
+            .map(|(k, v)| (k.to_vec().into(), v.to_vec().into()));
+
+        Box::new(iter)
+    }
+
+    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
+        let prefix = prefix.to_vec();
+        Box::pin(async move {
+            self.0.watch_prefix(prefix).await;
+        })
+    }
+}
@@ -0,0 +1,392 @@
+use super::{DatabaseEngine, Tree};
+use crate::{database::Config, Result};
+use parking_lot::{Mutex, MutexGuard, RwLock};
+use rusqlite::{Connection, DatabaseName::Main, OptionalExtension};
+use std::{
+    cell::RefCell,
+    collections::{hash_map, HashMap},
+    future::Future,
+    path::{Path, PathBuf},
+    pin::Pin,
+    sync::Arc,
+};
+use thread_local::ThreadLocal;
+use tokio::sync::watch;
+use tracing::debug;
+
+thread_local! {
+    static READ_CONNECTION: RefCell<Option<&'static Connection>> = RefCell::new(None);
+    static READ_CONNECTION_ITERATOR: RefCell<Option<&'static Connection>> = RefCell::new(None);
+}
+
+struct PreparedStatementIterator<'a> {
+    pub iterator: Box<dyn Iterator<Item = TupleOfBytes> + 'a>,
+    pub statement_ref: NonAliasingBox<rusqlite::Statement<'a>>,
+}
+
+impl Iterator for PreparedStatementIterator<'_> {
+    type Item = TupleOfBytes;
+
+    fn next(&mut self) -> Option<Self::Item> {
+        self.iterator.next()
+    }
+}
+
+struct NonAliasingBox<T>(*mut T);
+impl<T> Drop for NonAliasingBox<T> {
+    fn drop(&mut self) {
+        unsafe { Box::from_raw(self.0) };
+    }
+}
+
+pub struct Engine {
+    writer: Mutex<Connection>,
+    read_conn_tls: ThreadLocal<Connection>,
+    read_iterator_conn_tls: ThreadLocal<Connection>,
+
+    path: PathBuf,
+    cache_size_per_thread: u32,
+}
+
+impl Engine {
+    fn prepare_conn(path: &Path, cache_size_kb: u32) -> Result<Connection> {
+        let conn = Connection::open(&path)?;
+
+        conn.pragma_update(Some(Main), "page_size", &2048)?;
+        conn.pragma_update(Some(Main), "journal_mode", &"WAL")?;
+        conn.pragma_update(Some(Main), "synchronous", &"NORMAL")?;
+        conn.pragma_update(Some(Main), "cache_size", &(-i64::from(cache_size_kb)))?;
+        conn.pragma_update(Some(Main), "wal_autocheckpoint", &0)?;
+
+        Ok(conn)
+    }
+
+    fn write_lock(&self) -> MutexGuard<'_, Connection> {
+        self.writer.lock()
+    }
+
+    fn read_lock(&self) -> &Connection {
+        self.read_conn_tls
+            .get_or(|| Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap())
+    }
+
+    fn read_lock_iterator(&self) -> &Connection {
+        self.read_iterator_conn_tls
+            .get_or(|| Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap())
+    }
+
+    pub fn flush_wal(self: &Arc<Self>) -> Result<()> {
+        self.write_lock()
+            .pragma_update(Some(Main), "wal_checkpoint", &"RESTART")?;
+        Ok(())
+    }
+}
+
+impl DatabaseEngine for Engine {
+    fn open(config: &Config) -> Result<Arc<Self>> {
+        let path = Path::new(&config.database_path).join("conduit.db");
+
+        // calculates cache-size per permanent connection
+        // 1. convert MB to KiB
+        // 2. divide by permanent connections + permanent iter connections + write connection
+        // 3. round down to nearest integer
+        let cache_size_per_thread: u32 = ((config.db_cache_capacity_mb * 1024.0)
+            / ((num_cpus::get().max(1) * 2) + 1) as f64)
+            as u32;
+
+        let writer = Mutex::new(Self::prepare_conn(&path, cache_size_per_thread)?);
+
+        let arc = Arc::new(Engine {
+            writer,
+            read_conn_tls: ThreadLocal::new(),
+            read_iterator_conn_tls: ThreadLocal::new(),
+            path,
+            cache_size_per_thread,
+        });
+
+        Ok(arc)
+    }
+
+    fn open_tree(self: &Arc<Self>, name: &str) -> Result<Arc<dyn Tree>> {
+        self.write_lock().execute(&format!("CREATE TABLE IF NOT EXISTS {} ( \"key\" BLOB PRIMARY KEY, \"value\" BLOB NOT NULL )", name), [])?;
+
+        Ok(Arc::new(SqliteTable {
+            engine: Arc::clone(self),
+            name: name.to_owned(),
+            watchers: RwLock::new(HashMap::new()),
+        }))
+    }
+
+    fn flush(self: &Arc<Self>) -> Result<()> {
+        // we enabled PRAGMA synchronous=normal, so this should not be necessary
+        Ok(())
+    }
+}
+
+pub struct SqliteTable {
+    engine: Arc<Engine>,
+    name: String,
+    watchers: RwLock<HashMap<Vec<u8>, (watch::Sender<()>, watch::Receiver<()>)>>,
+}
+
+type TupleOfBytes = (Vec<u8>, Vec<u8>);
+
+impl SqliteTable {
+    #[tracing::instrument(skip(self, guard, key))]
+    fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        //dbg!(&self.name);
+        Ok(guard
+            .prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())?
+            .query_row([key], |row| row.get(0))
+            .optional()?)
+    }
+
+    #[tracing::instrument(skip(self, guard, key, value))]
+    fn insert_with_guard(&self, guard: &Connection, key: &[u8], value: &[u8]) -> Result<()> {
+        //dbg!(&self.name);
+        guard.execute(
+            format!(
+                "INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)",
+                self.name
+            )
+            .as_str(),
+            [key, value],
+        )?;
+        Ok(())
+    }
+
+    pub fn iter_with_guard<'a>(
+        &'a self,
+        guard: &'a Connection,
+    ) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
+        let statement = Box::leak(Box::new(
+            guard
+                .prepare(&format!(
+                    "SELECT key, value FROM {} ORDER BY key ASC",
+                    &self.name
+                ))
+                .unwrap(),
+        ));
+
+        let statement_ref = NonAliasingBox(statement);
+
+        //let name = self.name.clone();
+
+        let iterator = Box::new(
+            statement
+                .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
+                .unwrap()
+                .map(move |r| {
+                    //dbg!(&name);
+                    r.unwrap()
+                }),
+        );
+
+        Box::new(PreparedStatementIterator {
+            iterator,
+            statement_ref,
+        })
+    }
+}
+
+impl Tree for SqliteTable {
+    #[tracing::instrument(skip(self, key))]
+    fn get(&self, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        self.get_with_guard(self.engine.read_lock(), key)
+    }
+
+    #[tracing::instrument(skip(self, key, value))]
+    fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
+        let guard = self.engine.write_lock();
+        self.insert_with_guard(&guard, key, value)?;
+        drop(guard);
+
+        let watchers = self.watchers.read();
+        let mut triggered = Vec::new();
+
+        for length in 0..=key.len() {
+            if watchers.contains_key(&key[..length]) {
+                triggered.push(&key[..length]);
+            }
+        }
+
+        drop(watchers);
+
+        if !triggered.is_empty() {
+            let mut watchers = self.watchers.write();
+            for prefix in triggered {
+                if let Some(tx) = watchers.remove(prefix) {
+                    let _ = tx.0.send(());
+                }
+            }
+        };
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, iter))]
+    fn insert_batch<'a>(&self, iter: &mut dyn Iterator<Item = (Vec<u8>, Vec<u8>)>) -> Result<()> {
+        let guard = self.engine.write_lock();
+
+        guard.execute("BEGIN", [])?;
+        for (key, value) in iter {
+            self.insert_with_guard(&guard, &key, &value)?;
+        }
+        guard.execute("COMMIT", [])?;
+
+        drop(guard);
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, iter))]
+    fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
+        let guard = self.engine.write_lock();
+
+        guard.execute("BEGIN", [])?;
+        for key in iter {
+            let old = self.get_with_guard(&guard, &key)?;
+            let new = crate::utils::increment(old.as_deref())
+                .expect("utils::increment always returns Some");
+            self.insert_with_guard(&guard, &key, &new)?;
+        }
+        guard.execute("COMMIT", [])?;
+
+        drop(guard);
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, key))]
+    fn remove(&self, key: &[u8]) -> Result<()> {
+        let guard = self.engine.write_lock();
+
+        guard.execute(
+            format!("DELETE FROM {} WHERE key = ?", self.name).as_str(),
+            [key],
+        )?;
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
+        let guard = self.engine.read_lock_iterator();
+
+        self.iter_with_guard(guard)
+    }
+
+    #[tracing::instrument(skip(self, from, backwards))]
+    fn iter_from<'a>(
+        &'a self,
+        from: &[u8],
+        backwards: bool,
+    ) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
+        let guard = self.engine.read_lock_iterator();
+        let from = from.to_vec(); // TODO change interface?
+
+        //let name = self.name.clone();
+
+        if backwards {
+            let statement = Box::leak(Box::new(
+                guard
+                    .prepare(&format!(
+                        "SELECT key, value FROM {} WHERE key <= ? ORDER BY key DESC",
+                        &self.name
+                    ))
+                    .unwrap(),
+            ));
+
+            let statement_ref = NonAliasingBox(statement);
+
+            let iterator = Box::new(
+                statement
+                    .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
+                    .unwrap()
+                    .map(move |r| {
+                        //dbg!(&name);
+                        r.unwrap()
+                    }),
+            );
+            Box::new(PreparedStatementIterator {
+                iterator,
+                statement_ref,
+            })
+        } else {
+            let statement = Box::leak(Box::new(
+                guard
+                    .prepare(&format!(
+                        "SELECT key, value FROM {} WHERE key >= ? ORDER BY key ASC",
+                        &self.name
+                    ))
+                    .unwrap(),
+            ));
+
+            let statement_ref = NonAliasingBox(statement);
+
+            let iterator = Box::new(
+                statement
+                    .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
+                    .unwrap()
+                    .map(move |r| {
+                        //dbg!(&name);
+                        r.unwrap()
+                    }),
+            );
+
+            Box::new(PreparedStatementIterator {
+                iterator,
+                statement_ref,
+            })
+        }
+    }
+
+    #[tracing::instrument(skip(self, key))]
+    fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
+        let guard = self.engine.write_lock();
+
+        let old = self.get_with_guard(&guard, key)?;
+
+        let new =
+            crate::utils::increment(old.as_deref()).expect("utils::increment always returns Some");
+
+        self.insert_with_guard(&guard, key, &new)?;
+
+        Ok(new)
+    }
+
+    #[tracing::instrument(skip(self, prefix))]
+    fn scan_prefix<'a>(&'a self, prefix: Vec<u8>) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
+        Box::new(
+            self.iter_from(&prefix, false)
+                .take_while(move |(key, _)| key.starts_with(&prefix)),
+        )
+    }
+
+    #[tracing::instrument(skip(self, prefix))]
+    fn watch_prefix<'a>(&'a self, prefix: &[u8]) -> Pin<Box<dyn Future<Output = ()> + Send + 'a>> {
+        let mut rx = match self.watchers.write().entry(prefix.to_vec()) {
+            hash_map::Entry::Occupied(o) => o.get().1.clone(),
+            hash_map::Entry::Vacant(v) => {
+                let (tx, rx) = tokio::sync::watch::channel(());
+                v.insert((tx, rx.clone()));
+                rx
+            }
+        };
+
+        Box::pin(async move {
+            // Tx is never destroyed
+            rx.changed().await.unwrap();
+        })
+    }
+
+    #[tracing::instrument(skip(self))]
+    fn clear(&self) -> Result<()> {
+        debug!("clear: running");
+        self.engine
+            .write_lock()
+            .execute(format!("DELETE FROM {}", self.name).as_str(), [])?;
+        debug!("clear: ran");
+        Ok(())
+    }
+}
@@ -1,21 +1,23 @@
 use crate::{utils, Error, Result};
 use ruma::{
    api::client::error::ErrorKind,
-    events::{AnyEvent as EduEvent, EventType},
+    events::{AnyEphemeralRoomEvent, EventType},
    serde::Raw,
    RoomId, UserId,
 };
 use serde::{de::DeserializeOwned, Serialize};
-use sled::IVec;
-use std::{collections::HashMap, convert::TryFrom};
+use std::{collections::HashMap, convert::TryFrom, sync::Arc};
+
+use super::abstraction::Tree;

-#[derive(Clone)]
 pub struct AccountData {
-    pub(super) roomuserdataid_accountdata: sled::Tree, // RoomUserDataId = Room + User + Count + Type
+    pub(super) roomuserdataid_accountdata: Arc<dyn Tree>, // RoomUserDataId = Room + User + Count + Type
+    pub(super) roomusertype_roomuserdataid: Arc<dyn Tree>, // RoomUserType = Room + User + Type
 }

 impl AccountData {
    /// Places one event in the account data of the user and removes the previous entry.
+    #[tracing::instrument(skip(self, room_id, user_id, event_type, data, globals))]
    pub fn update<T: Serialize>(
        &self,
        room_id: Option<&RoomId>,
@@ -30,19 +32,16 @@ impl AccountData {
            .as_bytes()
            .to_vec();
        prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
+        prefix.extend_from_slice(user_id.as_bytes());
        prefix.push(0xff);

-        // Remove old entry
-        if let Some(previous) = self.find_event(room_id, user_id, &event_type) {
-            let (old_key, _) = previous?;
-            self.roomuserdataid_accountdata.remove(old_key)?;
-        }
+        let mut roomuserdataid = prefix.clone();
+        roomuserdataid.extend_from_slice(&globals.next_count()?.to_be_bytes());
+        roomuserdataid.push(0xff);
+        roomuserdataid.extend_from_slice(event_type.as_bytes());

        let mut key = prefix;
-        key.extend_from_slice(&globals.next_count()?.to_be_bytes());
-        key.push(0xff);
-        key.extend_from_slice(event_type.to_string().as_bytes());
+        key.extend_from_slice(event_type.as_bytes());

        let json = serde_json::to_value(data).expect("all types here can be serialized"); // TODO: maybe add error handling
        if json.get("type").is_none() || json.get("content").is_none() {
@@ -52,35 +51,65 @@ impl AccountData {
            ));
        }

-        self.roomuserdataid_accountdata
-            .insert(key, &*json.to_string())?;
+        self.roomuserdataid_accountdata.insert(
+            &roomuserdataid,
+            &serde_json::to_vec(&json).expect("to_vec always works on json values"),
+        )?;
+
+        let prev = self.roomusertype_roomuserdataid.get(&key)?;
+
+        self.roomusertype_roomuserdataid
+            .insert(&key, &roomuserdataid)?;
+
+        // Remove old entry
+        if let Some(prev) = prev {
+            self.roomuserdataid_accountdata.remove(&prev)?;
+        }

        Ok(())
    }

    /// Searches the account data for a specific kind.
+    #[tracing::instrument(skip(self, room_id, user_id, kind))]
    pub fn get<T: DeserializeOwned>(
        &self,
        room_id: Option<&RoomId>,
        user_id: &UserId,
        kind: EventType,
    ) -> Result<Option<T>> {
-        self.find_event(room_id, user_id, &kind)
-            .map(|r| {
-                let (_, v) = r?;
-                serde_json::from_slice(&v).map_err(|_| Error::bad_database("could not deserialize"))
+        let mut key = room_id
+            .map(|r| r.to_string())
+            .unwrap_or_default()
+            .as_bytes()
+            .to_vec();
+        key.push(0xff);
+        key.extend_from_slice(user_id.as_bytes());
+        key.push(0xff);
+        key.extend_from_slice(kind.as_ref().as_bytes());
+
+        self.roomusertype_roomuserdataid
+            .get(&key)?
+            .and_then(|roomuserdataid| {
+                self.roomuserdataid_accountdata
+                    .get(&roomuserdataid)
+                    .transpose()
+            })
+            .transpose()?
+            .map(|data| {
+                serde_json::from_slice(&data)
+                    .map_err(|_| Error::bad_database("could not deserialize"))
            })
            .transpose()
    }

    /// Returns all changes to the account data that happened after `since`.
-    #[tracing::instrument(skip(self))]
+    #[tracing::instrument(skip(self, room_id, user_id, since))]
    pub fn changes_since(
        &self,
        room_id: Option<&RoomId>,
        user_id: &UserId,
        since: u64,
-    ) -> Result<HashMap<EventType, Raw<EduEvent>>> {
+    ) -> Result<HashMap<EventType, Raw<AnyEphemeralRoomEvent>>> {
        let mut userdata = HashMap::new();

        let mut prefix = room_id
@@ -89,7 +118,7 @@ impl AccountData {
            .as_bytes()
            .to_vec();
        prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
+        prefix.extend_from_slice(user_id.as_bytes());
        prefix.push(0xff);

        // Skip the data that's exactly at since, because we sent that last time
@@ -98,8 +127,7 @@ impl AccountData {

        for r in self
            .roomuserdataid_accountdata
-            .range(&*first_possible..)
-            .filter_map(|r| r.ok())
+            .iter_from(&first_possible, false)
            .take_while(move |(k, _)| k.starts_with(&prefix))
            .map(|(k, v)| {
                Ok::<_, Error>((
@@ -110,7 +138,7 @@ impl AccountData {
                        .map_err(|_| Error::bad_database("RoomUserData ID in db is invalid."))?,
                    )
                    .map_err(|_| Error::bad_database("RoomUserData ID in db is invalid."))?,
-                    serde_json::from_slice::<Raw<EduEvent>>(&v).map_err(|_| {
+                    serde_json::from_slice::<Raw<AnyEphemeralRoomEvent>>(&v).map_err(|_| {
                        Error::bad_database("Database contains invalid account data.")
                    })?,
                ))
@@ -122,38 +150,4 @@ impl AccountData {

        Ok(userdata)
    }
-
-    fn find_event(
-        &self,
-        room_id: Option<&RoomId>,
-        user_id: &UserId,
-        kind: &EventType,
-    ) -> Option<Result<(IVec, IVec)>> {
-        let mut prefix = room_id
-            .map(|r| r.to_string())
-            .unwrap_or_default()
-            .as_bytes()
-            .to_vec();
-        prefix.push(0xff);
-        prefix.extend_from_slice(&user_id.to_string().as_bytes());
-        prefix.push(0xff);
-        let kind = kind.clone();
-
-        self.roomuserdataid_accountdata
-            .scan_prefix(prefix)
-            .rev()
-            .find(move |r| {
-                r.as_ref()
-                    .map(|(k, _)| {
-                        k.rsplit(|&b| b == 0xff)
-                            .next()
-                            .map(|current_event_type| {
-                                current_event_type == kind.to_string().as_bytes()
-                            })
-                            .unwrap_or(false)
-                    })
-                    .unwrap_or(false)
-            })
-            .map(|r| Ok(r?))
-    }
 }
@@ -1,12 +1,16 @@
-use std::convert::{TryFrom, TryInto};
+use std::{
+    convert::{TryFrom, TryInto},
+    sync::Arc,
+};

-use crate::pdu::PduBuilder;
-use log::warn;
+use crate::{pdu::PduBuilder, Database};
 use rocket::futures::{channel::mpsc, stream::StreamExt};
 use ruma::{
    events::{room::message, EventType},
    UserId,
 };
+use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard};
+use tracing::warn;

 pub enum AdminCommand {
    RegisterAppservice(serde_yaml::Value),
@@ -22,70 +26,97 @@ pub struct Admin {
 impl Admin {
    pub fn start_handler(
        &self,
-        db: super::Database,
+        db: Arc<RwLock<Database>>,
        mut receiver: mpsc::UnboundedReceiver<AdminCommand>,
    ) {
        tokio::spawn(async move {
            // TODO: Use futures when we have long admin commands
            //let mut futures = FuturesUnordered::new();

-            let conduit_user = UserId::try_from(format!("@conduit:{}", db.globals.server_name()))
-                .expect("@conduit:server_name is valid");
+            let guard = db.read().await;

-            let conduit_room = db
+            let conduit_user =
+                UserId::try_from(format!("@conduit:{}", guard.globals.server_name()))
+                    .expect("@conduit:server_name is valid");
+
+            let conduit_room = guard
                .rooms
                .id_from_alias(
-                    &format!("#admins:{}", db.globals.server_name())
+                    &format!("#admins:{}", guard.globals.server_name())
                        .try_into()
                        .expect("#admins:server_name is a valid room alias"),
                )
                .unwrap();

-            if conduit_room.is_none() {
-                warn!("Conduit instance does not have an #admins room. Logging to that room will not work. Restart Conduit after creating a user to fix this.");
-            }
-
-            let send_message = |message: message::MessageEventContent| {
-                if let Some(conduit_room) = &conduit_room {
-                    db.rooms
-                        .build_and_append_pdu(
-                            PduBuilder {
-                                event_type: EventType::RoomMessage,
-                                content: serde_json::to_value(message)
-                                    .expect("event is valid, we just created it"),
-                                unsigned: None,
-                                state_key: None,
-                                redacts: None,
-                            },
-                            &conduit_user,
-                            &conduit_room,
-                            &db,
-                        )
-                        .unwrap();
+            let conduit_room = match conduit_room {
+                None => {
+                    warn!("Conduit instance does not have an #admins room. Logging to that room will not work. Restart Conduit after creating a user to fix this.");
+                    return;
                }
+                Some(r) => r,
+            };
+
+            drop(guard);
+
+            let send_message = |message: message::MessageEventContent,
+                                guard: RwLockReadGuard<'_, Database>,
+                                mutex_lock: &MutexGuard<'_, ()>| {
+                guard
+                    .rooms
+                    .build_and_append_pdu(
+                        PduBuilder {
+                            event_type: EventType::RoomMessage,
+                            content: serde_json::to_value(message)
+                                .expect("event is valid, we just created it"),
+                            unsigned: None,
+                            state_key: None,
+                            redacts: None,
+                        },
+                        &conduit_user,
+                        &conduit_room,
+                        &guard,
+                        mutex_lock,
+                    )
+                    .unwrap();
            };

            loop {
                tokio::select! {
                    Some(event) = receiver.next() => {
+                        let guard = db.read().await;
+                        let mutex_state = Arc::clone(
+                            guard.globals
+                                .roomid_mutex_state
+                                .write()
+                                .unwrap()
+                                .entry(conduit_room.clone())
+                                .or_default(),
+                        );
+                        let state_lock = mutex_state.lock().await;
+
                        match event {
                            AdminCommand::RegisterAppservice(yaml) => {
-                                db.appservice.register_appservice(yaml).unwrap(); // TODO handle error
+                                guard.appservice.register_appservice(yaml).unwrap(); // TODO handle error
                            }
                            AdminCommand::ListAppservices => {
-                                let appservices = db.appservice.iter_ids().collect::<Vec<_>>();
-                                let count = appservices.len();
-                                let output = format!(
-                                    "Appservices ({}): {}",
-                                    count,
-                                    appservices.into_iter().filter_map(|r| r.ok()).collect::<Vec<_>>().join(", ")
-                                );
-                                send_message(message::MessageEventContent::text_plain(output));
+                                if let Ok(appservices) = guard.appservice.iter_ids().map(|ids| ids.collect::<Vec<_>>()) {
+                                    let count = appservices.len();
+                                    let output = format!(
+                                        "Appservices ({}): {}",
+                                        count,
+                                        appservices.into_iter().filter_map(|r| r.ok()).collect::<Vec<_>>().join(", ")
+                                    );
+                                    send_message(message::MessageEventContent::text_plain(output), guard, &state_lock);
+                                } else {
+                                    send_message(message::MessageEventContent::text_plain("Failed to get appservices."), guard, &state_lock);
+                                }
                            }
                            AdminCommand::SendMessage(message) => {
-                                send_message(message);
+                                send_message(message, guard, &state_lock);
                            }
                        }
+
+                        drop(state_lock);
                    }
                }
            }
@@ -93,6 +124,6 @@ impl Admin {
    }

    pub fn send(&self, command: AdminCommand) {
-        self.sender.unbounded_send(command).unwrap()
+        self.sender.unbounded_send(command).unwrap();
    }
 }
@@ -4,18 +4,21 @@ use std::{
    sync::{Arc, RwLock},
 };

-#[derive(Clone)]
+use super::abstraction::Tree;
+
 pub struct Appservice {
    pub(super) cached_registrations: Arc<RwLock<HashMap<String, serde_yaml::Value>>>,
-    pub(super) id_appserviceregistrations: sled::Tree,
+    pub(super) id_appserviceregistrations: Arc<dyn Tree>,
 }

 impl Appservice {
    pub fn register_appservice(&self, yaml: serde_yaml::Value) -> Result<()> {
        // TODO: Rumaify
        let id = yaml.get("id").unwrap().as_str().unwrap();
-        self.id_appserviceregistrations
-            .insert(id, serde_yaml::to_string(&yaml).unwrap().as_bytes())?;
+        self.id_appserviceregistrations.insert(
+            id.as_bytes(),
+            serde_yaml::to_string(&yaml).unwrap().as_bytes(),
+        )?;
        self.cached_registrations
            .write()
            .unwrap()
@@ -31,39 +34,38 @@ impl Appservice {
            .get(id)
            .map_or_else(
                || {
-                    Ok(self
-                        .id_appserviceregistrations
-                        .get(id)?
+                    self.id_appserviceregistrations
+                        .get(id.as_bytes())?
                        .map(|bytes| {
-                            Ok::<_, Error>(serde_yaml::from_slice(&bytes).map_err(|_| {
+                            serde_yaml::from_slice(&bytes).map_err(|_| {
                                Error::bad_database(
                                    "Invalid registration bytes in id_appserviceregistrations.",
                                )
-                            })?)
+                            })
                        })
-                        .transpose()?)
+                        .transpose()
                },
                |r| Ok(Some(r.clone())),
            )
    }

-    pub fn iter_ids(&self) -> impl Iterator<Item = Result<String>> {
-        self.id_appserviceregistrations.iter().keys().map(|id| {
-            Ok(utils::string_from_bytes(&id?).map_err(|_| {
-                Error::bad_database("Invalid id bytes in id_appserviceregistrations.")
-            })?)
-        })
+    pub fn iter_ids(&self) -> Result<impl Iterator<Item = Result<String>> + '_> {
+        Ok(self.id_appserviceregistrations.iter().map(|(id, _)| {
+            utils::string_from_bytes(&id)
+                .map_err(|_| Error::bad_database("Invalid id bytes in id_appserviceregistrations."))
+        }))
    }

-    pub fn iter_all<'a>(
-        &'a self,
-    ) -> impl Iterator<Item = Result<(String, serde_yaml::Value)>> + 'a {
-        self.iter_ids().filter_map(|id| id.ok()).map(move |id| {
-            Ok((
-                id.clone(),
-                self.get_registration(&id)?
-                    .expect("iter_ids only returns appservices that exist"),
-            ))
-        })
+    pub fn all(&self) -> Result<Vec<(String, serde_yaml::Value)>> {
+        self.iter_ids()?
+            .filter_map(|id| id.ok())
+            .map(move |id| {
+                Ok((
+                    id.clone(),
+                    self.get_registration(&id)?
+                        .expect("iter_ids only returns appservices that exist"),
+                ))
+            })
+            .collect()
    }
 }
@@ -1,43 +1,102 @@
-use crate::{database::Config, utils, Error, Result};
-use log::error;
+use crate::{database::Config, server_server::FedDest, utils, ConduitResult, Error, Result};
 use ruma::{
-    api::federation::discovery::{ServerSigningKeys, VerifyKey},
-    ServerName, ServerSigningKeyId,
+    api::{
+        client::r0::sync::sync_events,
+        federation::discovery::{ServerSigningKeys, VerifyKey},
+    },
+    DeviceId, EventId, MilliSecondsSinceUnixEpoch, RoomId, ServerName, ServerSigningKeyId, UserId,
 };
 use std::{
    collections::{BTreeMap, HashMap},
-    sync::{Arc, RwLock},
-    time::Duration,
+    fs,
+    future::Future,
+    net::IpAddr,
+    path::PathBuf,
+    sync::{Arc, Mutex, RwLock},
+    time::{Duration, Instant},
 };
+use tokio::sync::{broadcast, watch::Receiver, Mutex as TokioMutex, Semaphore};
+use tracing::error;
 use trust_dns_resolver::TokioAsyncResolver;

-pub const COUNTER: &str = "c";
+use super::abstraction::Tree;

-type WellKnownMap = HashMap<Box<ServerName>, (String, Option<String>)>;
+pub const COUNTER: &[u8] = b"c";
+
+type WellKnownMap = HashMap<Box<ServerName>, (FedDest, String)>;
+type TlsNameMap = HashMap<String, (Vec<IpAddr>, u16)>;
+type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries
+type SyncHandle = (
+    Option<String>,                                         // since
+    Receiver<Option<ConduitResult<sync_events::Response>>>, // rx
+);

-#[derive(Clone)]
 pub struct Globals {
    pub actual_destination_cache: Arc<RwLock<WellKnownMap>>, // actual_destination, host
-    pub(super) globals: sled::Tree,
+    pub tls_name_override: Arc<RwLock<TlsNameMap>>,
+    pub(super) globals: Arc<dyn Tree>,
    config: Config,
    keypair: Arc<ruma::signatures::Ed25519KeyPair>,
-    reqwest_client: reqwest::Client,
    dns_resolver: TokioAsyncResolver,
    jwt_decoding_key: Option<jsonwebtoken::DecodingKey<'static>>,
-    pub(super) servertimeout_signingkey: sled::Tree, // ServerName + Timeout Timestamp -> algorithm:key + pubkey
+    pub(super) server_signingkeys: Arc<dyn Tree>,
+    pub bad_event_ratelimiter: Arc<RwLock<HashMap<EventId, RateLimitState>>>,
+    pub bad_signature_ratelimiter: Arc<RwLock<HashMap<Vec<String>, RateLimitState>>>,
+    pub servername_ratelimiter: Arc<RwLock<HashMap<Box<ServerName>, Arc<Semaphore>>>>,
+    pub sync_receivers: RwLock<HashMap<(UserId, Box<DeviceId>), SyncHandle>>,
+    pub roomid_mutex_insert: RwLock<HashMap<RoomId, Arc<Mutex<()>>>>,
+    pub roomid_mutex_state: RwLock<HashMap<RoomId, Arc<TokioMutex<()>>>>,
+    pub roomid_mutex_federation: RwLock<HashMap<RoomId, Arc<TokioMutex<()>>>>, // this lock will be held longer
+    pub rotate: RotationHandler,
+}
+
+/// Handles "rotation" of long-polling requests. "Rotation" in this context is similar to "rotation" of log files and the like.
+///
+/// This is utilized to have sync workers return early and release read locks on the database.
+pub struct RotationHandler(broadcast::Sender<()>, broadcast::Receiver<()>);
+
+impl RotationHandler {
+    pub fn new() -> Self {
+        let (s, r) = broadcast::channel::<()>(1);
+
+        Self(s, r)
+    }
+
+    pub fn watch(&self) -> impl Future<Output = ()> {
+        let mut r = self.0.subscribe();
+
+        async move {
+            let _ = r.recv().await;
+        }
+    }
+
+    pub fn fire(&self) {
+        let _ = self.0.send(());
+    }
+}
+
+impl Default for RotationHandler {
+    fn default() -> Self {
+        Self::new()
+    }
 }

 impl Globals {
    pub fn load(
-        globals: sled::Tree,
-        servertimeout_signingkey: sled::Tree,
+        globals: Arc<dyn Tree>,
+        server_signingkeys: Arc<dyn Tree>,
        config: Config,
    ) -> Result<Self> {
-        let bytes = &*globals
-            .update_and_fetch("keypair", utils::generate_keypair)?
-            .expect("utils::generate_keypair always returns Some");
+        let keypair_bytes = globals.get(b"keypair")?.map_or_else(
+            || {
+                let keypair = utils::generate_keypair();
+                globals.insert(b"keypair", &keypair)?;
+                Ok::<_, Error>(keypair)
+            },
+            |s| Ok(s.to_vec()),
+        )?;

-        let mut parts = bytes.splitn(2, |&b| b == 0xff);
+        let mut parts = keypair_bytes.splitn(2, |&b| b == 0xff);

        let keypair = utils::string_from_bytes(
            // 1. version
@@ -54,7 +113,7 @@ impl Globals {
                .map(|key| (version, key))
        })
        .and_then(|(version, key)| {
-            ruma::signatures::Ed25519KeyPair::new(&key, version)
+            ruma::signatures::Ed25519KeyPair::from_der(key, version)
                .map_err(|_| Error::bad_database("Private or public keys are invalid."))
        });

@@ -62,35 +121,42 @@ impl Globals {
            Ok(k) => k,
            Err(e) => {
                error!("Keypair invalid. Deleting...");
-                globals.remove("keypair")?;
+                globals.remove(b"keypair")?;
                return Err(e);
            }
        };

-        let reqwest_client = reqwest::Client::builder()
-            .connect_timeout(Duration::from_secs(30))
-            .timeout(Duration::from_secs(60 * 3))
-            .pool_max_idle_per_host(1)
-            .build()
-            .unwrap();
+        let tls_name_override = Arc::new(RwLock::new(TlsNameMap::new()));

        let jwt_decoding_key = config
            .jwt_secret
            .as_ref()
            .map(|secret| jsonwebtoken::DecodingKey::from_secret(secret.as_bytes()).into_static());

-        Ok(Self {
+        let s = Self {
            globals,
            config,
            keypair: Arc::new(keypair),
-            reqwest_client,
            dns_resolver: TokioAsyncResolver::tokio_from_system_conf().map_err(|_| {
                Error::bad_config("Failed to set up trust dns resolver with system config.")
            })?,
-            actual_destination_cache: Arc::new(RwLock::new(HashMap::new())),
+            actual_destination_cache: Arc::new(RwLock::new(WellKnownMap::new())),
+            tls_name_override,
+            server_signingkeys,
            jwt_decoding_key,
-            servertimeout_signingkey,
-        })
+            bad_event_ratelimiter: Arc::new(RwLock::new(HashMap::new())),
+            bad_signature_ratelimiter: Arc::new(RwLock::new(HashMap::new())),
+            servername_ratelimiter: Arc::new(RwLock::new(HashMap::new())),
+            roomid_mutex_state: RwLock::new(HashMap::new()),
+            roomid_mutex_insert: RwLock::new(HashMap::new()),
+            roomid_mutex_federation: RwLock::new(HashMap::new()),
+            sync_receivers: RwLock::new(HashMap::new()),
+            rotate: RotationHandler::new(),
+        };
+
+        fs::create_dir_all(s.get_media_folder())?;
+
+        Ok(s)
    }

    /// Returns this server's keypair.
@@ -99,24 +165,29 @@ impl Globals {
    }

    /// Returns a reqwest client which can be used to send requests.
-    pub fn reqwest_client(&self) -> &reqwest::Client {
-        &self.reqwest_client
+    pub fn reqwest_client(&self) -> Result<reqwest::ClientBuilder> {
+        let mut reqwest_client_builder = reqwest::Client::builder()
+            .connect_timeout(Duration::from_secs(30))
+            .timeout(Duration::from_secs(60 * 3))
+            .pool_max_idle_per_host(1);
+        if let Some(proxy) = self.config.proxy.to_proxy()? {
+            reqwest_client_builder = reqwest_client_builder.proxy(proxy);
+        }
+
+        Ok(reqwest_client_builder)
    }

+    #[tracing::instrument(skip(self))]
    pub fn next_count(&self) -> Result<u64> {
-        Ok(utils::u64_from_bytes(
-            &self
-                .globals
-                .update_and_fetch(COUNTER, utils::increment)?
-                .expect("utils::increment will always put in a value"),
-        )
-        .map_err(|_| Error::bad_database("Count has invalid bytes."))?)
+        utils::u64_from_bytes(&self.globals.increment(COUNTER)?)
+            .map_err(|_| Error::bad_database("Count has invalid bytes."))
    }

+    #[tracing::instrument(skip(self))]
    pub fn current_count(&self) -> Result<u64> {
        self.globals.get(COUNTER)?.map_or(Ok(0_u64), |bytes| {
-            Ok(utils::u64_from_bytes(&bytes)
-                .map_err(|_| Error::bad_database("Count has invalid bytes."))?)
+            utils::u64_from_bytes(&bytes)
+                .map_err(|_| Error::bad_database("Count has invalid bytes."))
        })
    }

@@ -156,33 +227,43 @@ impl Globals {
    /// Remove the outdated keys and insert the new ones.
    ///
    /// This doesn't actually check that the keys provided are newer than the old set.
-    pub fn add_signing_key(&self, origin: &ServerName, keys: &ServerSigningKeys) -> Result<()> {
-        let mut key1 = origin.as_bytes().to_vec();
-        key1.push(0xff);
+    pub fn add_signing_key(
+        &self,
+        origin: &ServerName,
+        new_keys: ServerSigningKeys,
+    ) -> Result<BTreeMap<ServerSigningKeyId, VerifyKey>> {
+        // Not atomic, but this is not critical
+        let signingkeys = self.server_signingkeys.get(origin.as_bytes())?;

-        let mut key2 = key1.clone();
+        let mut keys = signingkeys
+            .and_then(|keys| serde_json::from_slice(&keys).ok())
+            .unwrap_or_else(|| {
+                // Just insert "now", it doesn't matter
+                ServerSigningKeys::new(origin.to_owned(), MilliSecondsSinceUnixEpoch::now())
+            });

-        let ts = keys
-            .valid_until_ts
-            .duration_since(std::time::UNIX_EPOCH)
-            .expect("time is valid")
-            .as_millis() as u64;
+        let ServerSigningKeys {
+            verify_keys,
+            old_verify_keys,
+            ..
+        } = new_keys;

-        key1.extend_from_slice(&ts.to_be_bytes());
-        key2.extend_from_slice(&(ts + 1).to_be_bytes());
+        keys.verify_keys.extend(verify_keys.into_iter());
+        keys.old_verify_keys.extend(old_verify_keys.into_iter());

-        self.servertimeout_signingkey.insert(
-            key1,
-            serde_json::to_vec(&keys.verify_keys).expect("ServerSigningKeys are a valid string"),
+        self.server_signingkeys.insert(
+            origin.as_bytes(),
+            &serde_json::to_vec(&keys).expect("serversigningkeys can be serialized"),
        )?;

-        self.servertimeout_signingkey.insert(
-            key2,
-            serde_json::to_vec(&keys.old_verify_keys)
-                .expect("ServerSigningKeys are a valid string"),
-        )?;
+        let mut tree = keys.verify_keys;
+        tree.extend(
+            keys.old_verify_keys
+                .into_iter()
+                .map(|old| (old.0, VerifyKey::new(old.1.key))),
+        );

-        Ok(())
+        Ok(tree)
    }

    /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found for the server.
@@ -190,25 +271,49 @@ impl Globals {
        &self,
        origin: &ServerName,
    ) -> Result<BTreeMap<ServerSigningKeyId, VerifyKey>> {
-        let mut response = BTreeMap::new();
+        let signingkeys = self
+            .server_signingkeys
+            .get(origin.as_bytes())?
+            .and_then(|bytes| serde_json::from_slice::<ServerSigningKeys>(&bytes).ok())
+            .map(|keys| {
+                let mut tree = keys.verify_keys;
+                tree.extend(
+                    keys.old_verify_keys
+                        .into_iter()
+                        .map(|old| (old.0, VerifyKey::new(old.1.key))),
+                );
+                tree
+            })
+            .unwrap_or_else(BTreeMap::new);

-        let now = crate::utils::millis_since_unix_epoch();
+        Ok(signingkeys)
+    }

-        for item in self.servertimeout_signingkey.scan_prefix(origin.as_bytes()) {
-            let (k, bytes) = item?;
-            let valid_until = k
-                .splitn(2, |&b| b == 0xff)
-                .nth(1)
-                .map(crate::utils::u64_from_bytes)
-                .ok_or_else(|| Error::bad_database("Invalid signing keys."))?
-                .map_err(|_| Error::bad_database("Invalid signing key valid until bytes"))?;
-            // If these keys are still valid use em!
-            if valid_until > now {
-                let btree: BTreeMap<_, _> = serde_json::from_slice(&bytes)
-                    .map_err(|_| Error::bad_database("Invalid BTreeMap<> of signing keys"))?;
-                response.extend(btree);
-            }
-        }
-        Ok(response)
+    pub fn database_version(&self) -> Result<u64> {
+        self.globals.get(b"version")?.map_or(Ok(0), |version| {
+            utils::u64_from_bytes(&version)
+                .map_err(|_| Error::bad_database("Database version id is invalid."))
+        })
+    }
+
+    pub fn bump_database_version(&self, new_version: u64) -> Result<()> {
+        self.globals
+            .insert(b"version", &new_version.to_be_bytes())?;
+        Ok(())
+    }
+
+    pub fn get_media_folder(&self) -> PathBuf {
+        let mut r = PathBuf::new();
+        r.push(self.config.database_path.clone());
+        r.push("media");
+        r
+    }
+
+    pub fn get_media_file(&self, key: &[u8]) -> PathBuf {
+        let mut r = PathBuf::new();
+        r.push(self.config.database_path.clone());
+        r.push("media");
+        r.push(base64::encode_config(key, base64::URL_SAFE_NO_PAD));
+        r
    }
 }
@@ -6,13 +6,14 @@ use ruma::{
    },
    RoomId, UserId,
 };
-use std::{collections::BTreeMap, convert::TryFrom};
+use std::{collections::BTreeMap, convert::TryFrom, sync::Arc};
+
+use super::abstraction::Tree;

-#[derive(Clone)]
 pub struct KeyBackups {
-    pub(super) backupid_algorithm: sled::Tree, // BackupId = UserId + Version(Count)
-    pub(super) backupid_etag: sled::Tree,      // BackupId = UserId + Version(Count)
-    pub(super) backupkeyid_backup: sled::Tree, // BackupKeyId = UserId + Version + RoomId + SessionId
+    pub(super) backupid_algorithm: Arc<dyn Tree>, // BackupId = UserId + Version(Count)
+    pub(super) backupid_etag: Arc<dyn Tree>,      // BackupId = UserId + Version(Count)
+    pub(super) backupkeyid_backup: Arc<dyn Tree>, // BackupKeyId = UserId + Version + RoomId + SessionId
 }

 impl KeyBackups {
@@ -24,14 +25,13 @@ impl KeyBackups {
    ) -> Result<String> {
        let version = globals.next_count()?.to_string();

-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&version.as_bytes());
+        key.extend_from_slice(version.as_bytes());

        self.backupid_algorithm.insert(
            &key,
-            &*serde_json::to_string(backup_metadata)
-                .expect("BackupAlgorithm::to_string always works"),
+            &serde_json::to_vec(backup_metadata).expect("BackupAlgorithm::to_vec always works"),
        )?;
        self.backupid_etag
            .insert(&key, &globals.next_count()?.to_be_bytes())?;
@@ -39,22 +39,17 @@ impl KeyBackups {
    }

    pub fn delete_backup(&self, user_id: &UserId, version: &str) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&version.as_bytes());
+        key.extend_from_slice(version.as_bytes());

        self.backupid_algorithm.remove(&key)?;
        self.backupid_etag.remove(&key)?;

        key.push(0xff);

-        for outdated_key in self
-            .backupkeyid_backup
-            .scan_prefix(&key)
-            .keys()
-            .filter_map(|r| r.ok())
-        {
-            self.backupkeyid_backup.remove(outdated_key)?;
+        for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
+            self.backupkeyid_backup.remove(&outdated_key)?;
        }

        Ok(())
@@ -67,9 +62,9 @@ impl KeyBackups {
        backup_metadata: &BackupAlgorithm,
        globals: &super::globals::Globals,
    ) -> Result<String> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&version.as_bytes());
+        key.extend_from_slice(version.as_bytes());

        if self.backupid_algorithm.get(&key)?.is_none() {
            return Err(Error::BadRequest(
@@ -80,22 +75,47 @@ impl KeyBackups {

        self.backupid_algorithm.insert(
            &key,
-            &*serde_json::to_string(backup_metadata)
-                .expect("BackupAlgorithm::to_string always works"),
+            serde_json::to_string(backup_metadata)
+                .expect("BackupAlgorithm::to_string always works")
+                .as_bytes(),
        )?;
        self.backupid_etag
            .insert(&key, &globals.next_count()?.to_be_bytes())?;
        Ok(version.to_string())
    }

-    pub fn get_latest_backup(&self, user_id: &UserId) -> Result<Option<(String, BackupAlgorithm)>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+    pub fn get_latest_backup_version(&self, user_id: &UserId) -> Result<Option<String>> {
+        let mut prefix = user_id.as_bytes().to_vec();
        prefix.push(0xff);
+        let mut last_possible_key = prefix.clone();
+        last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
+
        self.backupid_algorithm
-            .scan_prefix(&prefix)
-            .last()
-            .map_or(Ok(None), |r| {
-                let (key, value) = r?;
+            .iter_from(&last_possible_key, true)
+            .take_while(move |(k, _)| k.starts_with(&prefix))
+            .next()
+            .map_or(Ok(None), |(key, _)| {
+                utils::string_from_bytes(
+                    key.rsplit(|&b| b == 0xff)
+                        .next()
+                        .expect("rsplit always returns an element"),
+                )
+                .map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))
+                .map(Some)
+            })
+    }
+
+    pub fn get_latest_backup(&self, user_id: &UserId) -> Result<Option<(String, BackupAlgorithm)>> {
+        let mut prefix = user_id.as_bytes().to_vec();
+        prefix.push(0xff);
+        let mut last_possible_key = prefix.clone();
+        last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
+
+        self.backupid_algorithm
+            .iter_from(&last_possible_key, true)
+            .take_while(move |(k, _)| k.starts_with(&prefix))
+            .next()
+            .map_or(Ok(None), |(key, value)| {
                let version = utils::string_from_bytes(
                    key.rsplit(|&b| b == 0xff)
                        .next()
@@ -113,14 +133,16 @@ impl KeyBackups {
    }

    pub fn get_backup(&self, user_id: &UserId, version: &str) -> Result<Option<BackupAlgorithm>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
        key.extend_from_slice(version.as_bytes());

-        self.backupid_algorithm.get(key)?.map_or(Ok(None), |bytes| {
-            Ok(serde_json::from_slice(&bytes)
-                .map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid."))?)
-        })
+        self.backupid_algorithm
+            .get(&key)?
+            .map_or(Ok(None), |bytes| {
+                serde_json::from_slice(&bytes)
+                    .map_err(|_| Error::bad_database("Algorithm in backupid_algorithm is invalid."))
+            })
    }

    pub fn add_key(
@@ -132,7 +154,7 @@ impl KeyBackups {
        key_data: &KeyBackupData,
        globals: &super::globals::Globals,
    ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
        key.extend_from_slice(version.as_bytes());

@@ -147,30 +169,30 @@ impl KeyBackups {
            .insert(&key, &globals.next_count()?.to_be_bytes())?;

        key.push(0xff);
-        key.extend_from_slice(room_id.to_string().as_bytes());
+        key.extend_from_slice(room_id.as_bytes());
        key.push(0xff);
        key.extend_from_slice(session_id.as_bytes());

        self.backupkeyid_backup.insert(
            &key,
-            &*serde_json::to_string(&key_data).expect("KeyBackupData::to_string always works"),
+            &serde_json::to_vec(&key_data).expect("KeyBackupData::to_vec always works"),
        )?;

        Ok(())
    }

    pub fn count_keys(&self, user_id: &UserId, version: &str) -> Result<usize> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
        prefix.push(0xff);
        prefix.extend_from_slice(version.as_bytes());

-        Ok(self.backupkeyid_backup.scan_prefix(&prefix).count())
+        Ok(self.backupkeyid_backup.scan_prefix(prefix).count())
    }

    pub fn get_etag(&self, user_id: &UserId, version: &str) -> Result<String> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&version.as_bytes());
+        key.extend_from_slice(version.as_bytes());

        Ok(utils::u64_from_bytes(
            &self
@@ -187,40 +209,44 @@ impl KeyBackups {
        user_id: &UserId,
        version: &str,
    ) -> Result<BTreeMap<RoomId, RoomKeyBackup>> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+        let mut prefix = user_id.as_bytes().to_vec();
        prefix.push(0xff);
        prefix.extend_from_slice(version.as_bytes());
        prefix.push(0xff);

        let mut rooms = BTreeMap::<RoomId, RoomKeyBackup>::new();

-        for result in self.backupkeyid_backup.scan_prefix(&prefix).map(|r| {
-            let (key, value) = r?;
-            let mut parts = key.rsplit(|&b| b == 0xff);
+        for result in self
+            .backupkeyid_backup
+            .scan_prefix(prefix)
+            .map(|(key, value)| {
+                let mut parts = key.rsplit(|&b| b == 0xff);

-            let session_id = utils::string_from_bytes(
-                &parts
-                    .next()
-                    .ok_or_else(|| Error::bad_database("backupkeyid_backup key is invalid."))?,
-            )
-            .map_err(|_| Error::bad_database("backupkeyid_backup session_id is invalid."))?;
+                let session_id =
+                    utils::string_from_bytes(parts.next().ok_or_else(|| {
+                        Error::bad_database("backupkeyid_backup key is invalid.")
+                    })?)
+                    .map_err(|_| {
+                        Error::bad_database("backupkeyid_backup session_id is invalid.")
+                    })?;

-            let room_id = RoomId::try_from(
-                utils::string_from_bytes(
-                    &parts
-                        .next()
-                        .ok_or_else(|| Error::bad_database("backupkeyid_backup key is invalid."))?,
+                let room_id = RoomId::try_from(
+                    utils::string_from_bytes(parts.next().ok_or_else(|| {
+                        Error::bad_database("backupkeyid_backup key is invalid.")
+                    })?)
+                    .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid."))?,
                )
-                .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid."))?,
-            )
-            .map_err(|_| Error::bad_database("backupkeyid_backup room_id is invalid room id."))?;
+                .map_err(|_| {
+                    Error::bad_database("backupkeyid_backup room_id is invalid room id.")
+                })?;

-            let key_data = serde_json::from_slice(&value).map_err(|_| {
-                Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
-            })?;
+                let key_data = serde_json::from_slice(&value).map_err(|_| {
+                    Error::bad_database("KeyBackupData in backupkeyid_backup is invalid.")
+                })?;

-            Ok::<_, Error>((room_id, session_id, key_data))
-        }) {
+                Ok::<_, Error>((room_id, session_id, key_data))
+            })
+        {
            let (room_id, session_id, key_data) = result?;
            rooms
                .entry(room_id)
@@ -239,22 +265,22 @@ impl KeyBackups {
        user_id: &UserId,
        version: &str,
        room_id: &RoomId,
-    ) -> BTreeMap<String, KeyBackupData> {
-        let mut prefix = user_id.to_string().as_bytes().to_vec();
+    ) -> Result<BTreeMap<String, KeyBackupData>> {
+        let mut prefix = user_id.as_bytes().to_vec();
        prefix.push(0xff);
        prefix.extend_from_slice(version.as_bytes());
        prefix.push(0xff);
        prefix.extend_from_slice(room_id.as_bytes());
        prefix.push(0xff);

-        self.backupkeyid_backup
-            .scan_prefix(&prefix)
-            .map(|r| {
-                let (key, value) = r?;
+        Ok(self
+            .backupkeyid_backup
+            .scan_prefix(prefix)
+            .map(|(key, value)| {
                let mut parts = key.rsplit(|&b| b == 0xff);

                let session_id =
-                    utils::string_from_bytes(&parts.next().ok_or_else(|| {
+                    utils::string_from_bytes(parts.next().ok_or_else(|| {
                        Error::bad_database("backupkeyid_backup key is invalid.")
                    })?)
                    .map_err(|_| {
@@ -268,7 +294,7 @@ impl KeyBackups {
                Ok::<_, Error>((session_id, key_data))
            })
            .filter_map(|r| r.ok())
-            .collect()
+            .collect())
    }

    pub fn get_session(
@@ -278,7 +304,7 @@ impl KeyBackups {
        room_id: &RoomId,
        session_id: &str,
    ) -> Result<Option<KeyBackupData>> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
        key.extend_from_slice(version.as_bytes());
        key.push(0xff);
@@ -297,18 +323,13 @@ impl KeyBackups {
    }

    pub fn delete_all_keys(&self, user_id: &UserId, version: &str) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&version.as_bytes());
+        key.extend_from_slice(version.as_bytes());
        key.push(0xff);

-        for outdated_key in self
-            .backupkeyid_backup
-            .scan_prefix(&key)
-            .keys()
-            .filter_map(|r| r.ok())
-        {
-            self.backupkeyid_backup.remove(outdated_key)?;
+        for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
+            self.backupkeyid_backup.remove(&outdated_key)?;
        }

        Ok(())
@@ -320,20 +341,15 @@ impl KeyBackups {
        version: &str,
        room_id: &RoomId,
    ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&version.as_bytes());
+        key.extend_from_slice(version.as_bytes());
        key.push(0xff);
-        key.extend_from_slice(&room_id.as_bytes());
+        key.extend_from_slice(room_id.as_bytes());
        key.push(0xff);

-        for outdated_key in self
-            .backupkeyid_backup
-            .scan_prefix(&key)
-            .keys()
-            .filter_map(|r| r.ok())
-        {
-            self.backupkeyid_backup.remove(outdated_key)?;
+        for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
+            self.backupkeyid_backup.remove(&outdated_key)?;
        }

        Ok(())
@@ -346,21 +362,16 @@ impl KeyBackups {
        room_id: &RoomId,
        session_id: &str,
    ) -> Result<()> {
-        let mut key = user_id.to_string().as_bytes().to_vec();
+        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&version.as_bytes());
+        key.extend_from_slice(version.as_bytes());
        key.push(0xff);
-        key.extend_from_slice(&room_id.as_bytes());
+        key.extend_from_slice(room_id.as_bytes());
        key.push(0xff);
-        key.extend_from_slice(&session_id.as_bytes());
+        key.extend_from_slice(session_id.as_bytes());

-        for outdated_key in self
-            .backupkeyid_backup
-            .scan_prefix(&key)
-            .keys()
-            .filter_map(|r| r.ok())
-        {
-            self.backupkeyid_backup.remove(outdated_key)?;
+        for (outdated_key, _) in self.backupkeyid_backup.scan_prefix(key) {
+            self.backupkeyid_backup.remove(&outdated_key)?;
        }

        Ok(())
@@ -1,25 +1,31 @@
+use crate::database::globals::Globals;
 use image::{imageops::FilterType, GenericImageView};

+use super::abstraction::Tree;
 use crate::{utils, Error, Result};
-use std::mem;
+use std::{mem, sync::Arc};
+use tokio::{
+    fs::File,
+    io::{AsyncReadExt, AsyncWriteExt},
+};

 pub struct FileMeta {
-    pub filename: Option<String>,
+    pub content_disposition: Option<String>,
    pub content_type: Option<String>,
    pub file: Vec<u8>,
 }

-#[derive(Clone)]
 pub struct Media {
-    pub(super) mediaid_file: sled::Tree, // MediaId = MXC + WidthHeight + Filename + ContentType
+    pub(super) mediaid_file: Arc<dyn Tree>, // MediaId = MXC + WidthHeight + ContentDisposition + ContentType
 }

 impl Media {
-    /// Uploads or replaces a file.
-    pub fn create(
+    /// Uploads a file.
+    pub async fn create(
        &self,
        mxc: String,
-        filename: &Option<&str>,
+        globals: &Globals,
+        content_disposition: &Option<&str>,
        content_type: &Option<&str>,
        file: &[u8],
    ) -> Result<()> {
@@ -28,7 +34,12 @@ impl Media {
        key.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail
        key.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
        key.push(0xff);
-        key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default());
+        key.extend_from_slice(
+            content_disposition
+                .as_ref()
+                .map(|f| f.as_bytes())
+                .unwrap_or_default(),
+        );
        key.push(0xff);
        key.extend_from_slice(
            content_type
@@ -37,16 +48,21 @@ impl Media {
                .unwrap_or_default(),
        );

-        self.mediaid_file.insert(key, file)?;
+        let path = globals.get_media_file(&key);
+        let mut f = File::create(path).await?;
+        f.write_all(file).await?;

+        self.mediaid_file.insert(&key, &[])?;
        Ok(())
    }

    /// Uploads or replaces a file thumbnail.
-    pub fn upload_thumbnail(
+    #[allow(clippy::too_many_arguments)]
+    pub async fn upload_thumbnail(
        &self,
        mxc: String,
-        filename: &Option<String>,
+        globals: &Globals,
+        content_disposition: &Option<String>,
        content_type: &Option<String>,
        width: u32,
        height: u32,
@@ -57,7 +73,12 @@ impl Media {
        key.extend_from_slice(&width.to_be_bytes());
        key.extend_from_slice(&height.to_be_bytes());
        key.push(0xff);
-        key.extend_from_slice(filename.as_ref().map(|f| f.as_bytes()).unwrap_or_default());
+        key.extend_from_slice(
+            content_disposition
+                .as_ref()
+                .map(|f| f.as_bytes())
+                .unwrap_or_default(),
+        );
        key.push(0xff);
        key.extend_from_slice(
            content_type
@@ -66,48 +87,59 @@ impl Media {
                .unwrap_or_default(),
        );

-        self.mediaid_file.insert(key, file)?;
+        let path = globals.get_media_file(&key);
+        let mut f = File::create(path).await?;
+        f.write_all(file).await?;
+
+        self.mediaid_file.insert(&key, &[])?;

        Ok(())
    }

    /// Downloads a file.
-    pub fn get(&self, mxc: &str) -> Result<Option<FileMeta>> {
+    pub async fn get(&self, globals: &Globals, mxc: &str) -> Result<Option<FileMeta>> {
        let mut prefix = mxc.as_bytes().to_vec();
        prefix.push(0xff);
        prefix.extend_from_slice(&0_u32.to_be_bytes()); // Width = 0 if it's not a thumbnail
        prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
        prefix.push(0xff);

-        if let Some(r) = self.mediaid_file.scan_prefix(&prefix).next() {
-            let (key, file) = r?;
+        let first = self.mediaid_file.scan_prefix(prefix).next();
+        if let Some((key, _)) = first {
+            let path = globals.get_media_file(&key);
+            let mut file = Vec::new();
+            File::open(path).await?.read_to_end(&mut file).await?;
            let mut parts = key.rsplit(|&b| b == 0xff);

            let content_type = parts
                .next()
                .map(|bytes| {
-                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
+                    utils::string_from_bytes(bytes).map_err(|_| {
                        Error::bad_database("Content type in mediaid_file is invalid unicode.")
-                    })?)
+                    })
                })
                .transpose()?;

-            let filename_bytes = parts
+            let content_disposition_bytes = parts
                .next()
                .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?;

-            let filename = if filename_bytes.is_empty() {
+            let content_disposition = if content_disposition_bytes.is_empty() {
                None
            } else {
-                Some(utils::string_from_bytes(filename_bytes).map_err(|_| {
-                    Error::bad_database("Filename in mediaid_file is invalid unicode.")
-                })?)
+                Some(
+                    utils::string_from_bytes(content_disposition_bytes).map_err(|_| {
+                        Error::bad_database(
+                            "Content Disposition in mediaid_file is invalid unicode.",
+                        )
+                    })?,
+                )
            };

            Ok(Some(FileMeta {
-                filename,
+                content_disposition,
                content_type,
-                file: file.to_vec(),
+                file,
            }))
        } else {
            Ok(None)
@@ -137,7 +169,13 @@ impl Media {
    /// - Server creates the thumbnail and sends it to the user
    ///
    /// For width,height <= 96 the server uses another thumbnailing algorithm which crops the image afterwards.
-    pub fn get_thumbnail(&self, mxc: String, width: u32, height: u32) -> Result<Option<FileMeta>> {
+    pub async fn get_thumbnail(
+        &self,
+        mxc: String,
+        globals: &Globals,
+        width: u32,
+        height: u32,
+    ) -> Result<Option<FileMeta>> {
        let (width, height, crop) = self
            .thumbnail_properties(width, height)
            .unwrap_or((0, 0, false)); // 0, 0 because that's the original file
@@ -155,63 +193,74 @@ impl Media {
        original_prefix.extend_from_slice(&0_u32.to_be_bytes()); // Height = 0 if it's not a thumbnail
        original_prefix.push(0xff);

-        if let Some(r) = self.mediaid_file.scan_prefix(&thumbnail_prefix).next() {
+        let first_thumbnailprefix = self.mediaid_file.scan_prefix(thumbnail_prefix).next();
+        let first_originalprefix = self.mediaid_file.scan_prefix(original_prefix).next();
+        if let Some((key, _)) = first_thumbnailprefix {
            // Using saved thumbnail
-            let (key, file) = r?;
+            let path = globals.get_media_file(&key);
+            let mut file = Vec::new();
+            File::open(path).await?.read_to_end(&mut file).await?;
            let mut parts = key.rsplit(|&b| b == 0xff);

            let content_type = parts
                .next()
                .map(|bytes| {
-                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
+                    utils::string_from_bytes(bytes).map_err(|_| {
                        Error::bad_database("Content type in mediaid_file is invalid unicode.")
-                    })?)
+                    })
                })
                .transpose()?;

-            let filename_bytes = parts
+            let content_disposition_bytes = parts
                .next()
                .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?;

-            let filename = if filename_bytes.is_empty() {
+            let content_disposition = if content_disposition_bytes.is_empty() {
                None
            } else {
                Some(
-                    utils::string_from_bytes(filename_bytes)
-                        .map_err(|_| Error::bad_database("Filename in db is invalid."))?,
+                    utils::string_from_bytes(content_disposition_bytes).map_err(|_| {
+                        Error::bad_database("Content Disposition in db is invalid.")
+                    })?,
                )
            };

            Ok(Some(FileMeta {
-                filename,
+                content_disposition,
                content_type,
                file: file.to_vec(),
            }))
-        } else if let Some(r) = self.mediaid_file.scan_prefix(&original_prefix).next() {
+        } else if let Some((key, _)) = first_originalprefix {
            // Generate a thumbnail
+            let path = globals.get_media_file(&key);
+            let mut file = Vec::new();
+            File::open(path).await?.read_to_end(&mut file).await?;

-            let (key, file) = r?;
            let mut parts = key.rsplit(|&b| b == 0xff);

            let content_type = parts
                .next()
                .map(|bytes| {
-                    Ok::<_, Error>(utils::string_from_bytes(bytes).map_err(|_| {
+                    utils::string_from_bytes(bytes).map_err(|_| {
                        Error::bad_database("Content type in mediaid_file is invalid unicode.")
-                    })?)
+                    })
                })
                .transpose()?;

-            let filename_bytes = parts
+            let content_disposition_bytes = parts
                .next()
                .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?;

-            let filename = if filename_bytes.is_empty() {
+            let content_disposition = if content_disposition_bytes.is_empty() {
                None
            } else {
-                Some(utils::string_from_bytes(filename_bytes).map_err(|_| {
-                    Error::bad_database("Filename in mediaid_file is invalid unicode.")
-                })?)
+                Some(
+                    utils::string_from_bytes(content_disposition_bytes).map_err(|_| {
+                        Error::bad_database(
+                            "Content Disposition in mediaid_file is invalid unicode.",
+                        )
+                    })?,
+                )
            };

            if let Ok(image) = image::load_from_memory(&file) {
@@ -219,23 +268,24 @@ impl Media {
                let original_height = image.height();
                if width > original_width || height > original_height {
                    return Ok(Some(FileMeta {
-                        filename,
+                        content_disposition,
                        content_type,
                        file: file.to_vec(),
                    }));
                }

                let thumbnail = if crop {
-                    image.resize_to_fill(width, height, FilterType::Triangle)
+                    image.resize_to_fill(width, height, FilterType::CatmullRom)
                } else {
                    let (exact_width, exact_height) = {
                        // Copied from image::dynimage::resize_dimensions
                        let ratio = u64::from(original_width) * u64::from(height);
                        let nratio = u64::from(width) * u64::from(original_height);

-                        let use_width = nratio > ratio;
+                        let use_width = nratio <= ratio;
                        let intermediate = if use_width {
-                            u64::from(original_height) * u64::from(width) / u64::from(width)
+                            u64::from(original_height) * u64::from(width)
+                                / u64::from(original_width)
                        } else {
                            u64::from(original_width) * u64::from(height)
                                / u64::from(original_height)
@@ -282,17 +332,21 @@ impl Media {
                    widthheight,
                );

-                self.mediaid_file.insert(thumbnail_key, &*thumbnail_bytes)?;
+                let path = globals.get_media_file(&thumbnail_key);
+                let mut f = File::create(path).await?;
+                f.write_all(&thumbnail_bytes).await?;
+
+                self.mediaid_file.insert(&thumbnail_key, &[])?;

                Ok(Some(FileMeta {
-                    filename,
+                    content_disposition,
                    content_type,
                    file: thumbnail_bytes.to_vec(),
                }))
            } else {
                // Couldn't parse file to generate thumbnail, send original
                Ok(Some(FileMeta {
-                    filename,
+                    content_disposition,
                    content_type,
                    file: file.to_vec(),
                }))
@@ -0,0 +1,146 @@
+use reqwest::{Proxy, Url};
+use serde::Deserialize;
+
+use crate::Result;
+
+/// ## Examples:
+/// - No proxy (default):
+/// ```toml
+/// proxy ="none"
+/// ```
+/// - Global proxy
+/// ```toml
+/// [proxy]
+/// global = { url = "socks5h://localhost:9050" }
+/// ```
+/// - Proxy some domains
+/// ```toml
+/// [proxy]
+/// [[proxy.by_domain]]
+/// url = "socks5h://localhost:9050"
+/// include = ["*.onion", "matrix.myspecial.onion"]
+/// exclude = ["*.myspecial.onion"]
+/// ```
+/// ## Include vs. Exclude
+/// If include is an empty list, it is assumed to be `["*"]`.
+///
+/// If a domain matches both the exclude and include list, the proxy will only be used if it was
+/// included because of a more specific rule than it was excluded. In the above example, the proxy
+/// would be used for `ordinary.onion`, `matrix.myspecial.onion`, but not `hello.myspecial.onion`.
+#[derive(Clone, Debug, Deserialize)]
+#[serde(rename_all = "snake_case")]
+pub enum ProxyConfig {
+    None,
+    Global {
+        #[serde(deserialize_with = "crate::utils::deserialize_from_str")]
+        url: Url,
+    },
+    ByDomain(Vec<PartialProxyConfig>),
+}
+impl ProxyConfig {
+    pub fn to_proxy(&self) -> Result<Option<Proxy>> {
+        Ok(match self.clone() {
+            ProxyConfig::None => None,
+            ProxyConfig::Global { url } => Some(Proxy::all(url)?),
+            ProxyConfig::ByDomain(proxies) => Some(Proxy::custom(move |url| {
+                proxies.iter().find_map(|proxy| proxy.for_url(url)).cloned() // first matching proxy
+            })),
+        })
+    }
+}
+impl Default for ProxyConfig {
+    fn default() -> Self {
+        ProxyConfig::None
+    }
+}
+
+#[derive(Clone, Debug, Deserialize)]
+pub struct PartialProxyConfig {
+    #[serde(deserialize_with = "crate::utils::deserialize_from_str")]
+    url: Url,
+    #[serde(default)]
+    include: Vec<WildCardedDomain>,
+    #[serde(default)]
+    exclude: Vec<WildCardedDomain>,
+}
+impl PartialProxyConfig {
+    pub fn for_url(&self, url: &Url) -> Option<&Url> {
+        let domain = url.domain()?;
+        let mut included_because = None; // most specific reason it was included
+        let mut excluded_because = None; // most specific reason it was excluded
+        if self.include.is_empty() {
+            // treat empty include list as `*`
+            included_because = Some(&WildCardedDomain::WildCard)
+        }
+        for wc_domain in &self.include {
+            if wc_domain.matches(domain) {
+                match included_because {
+                    Some(prev) if !wc_domain.more_specific_than(prev) => (),
+                    _ => included_because = Some(wc_domain),
+                }
+            }
+        }
+        for wc_domain in &self.exclude {
+            if wc_domain.matches(domain) {
+                match excluded_because {
+                    Some(prev) if !wc_domain.more_specific_than(prev) => (),
+                    _ => excluded_because = Some(wc_domain),
+                }
+            }
+        }
+        match (included_because, excluded_because) {
+            (Some(a), Some(b)) if a.more_specific_than(b) => Some(&self.url), // included for a more specific reason than excluded
+            (Some(_), None) => Some(&self.url),
+            _ => None,
+        }
+    }
+}
+
+/// A domain name, that optionally allows a * as its first subdomain.
+#[derive(Clone, Debug)]
+pub enum WildCardedDomain {
+    WildCard,
+    WildCarded(String),
+    Exact(String),
+}
+impl WildCardedDomain {
+    pub fn matches(&self, domain: &str) -> bool {
+        match self {
+            WildCardedDomain::WildCard => true,
+            WildCardedDomain::WildCarded(d) => domain.ends_with(d),
+            WildCardedDomain::Exact(d) => domain == d,
+        }
+    }
+    pub fn more_specific_than(&self, other: &Self) -> bool {
+        match (self, other) {
+            (WildCardedDomain::WildCard, WildCardedDomain::WildCard) => false,
+            (_, WildCardedDomain::WildCard) => true,
+            (WildCardedDomain::Exact(a), WildCardedDomain::WildCarded(_)) => other.matches(a),
+            (WildCardedDomain::WildCarded(a), WildCardedDomain::WildCarded(b)) => {
+                a != b && a.ends_with(b)
+            }
+            _ => false,
+        }
+    }
+}
+impl std::str::FromStr for WildCardedDomain {
+    type Err = std::convert::Infallible;
+    fn from_str(s: &str) -> std::result::Result<Self, Self::Err> {
+        // maybe do some domain validation?
+        Ok(if s.starts_with("*.") {
+            WildCardedDomain::WildCarded(s[1..].to_owned())
+        } else if s == "*" {
+            WildCardedDomain::WildCarded("".to_owned())
+        } else {
+            WildCardedDomain::Exact(s.to_owned())
+        })
+    }
+}
+impl<'de> serde::de::Deserialize<'de> for WildCardedDomain {
+    fn deserialize<D>(deserializer: D) -> std::result::Result<Self, D::Error>
+    where
+        D: serde::de::Deserializer<'de>,
+    {
+        crate::utils::deserialize_from_str(deserializer)
+    }
+}
@@ -0,0 +1,344 @@
+use crate::{Database, Error, PduEvent, Result};
+use bytes::BytesMut;
+use ruma::{
+    api::{
+        client::r0::push::{get_pushers, set_pusher, PusherKind},
+        push_gateway::send_event_notification::{
+            self,
+            v1::{Device, Notification, NotificationCounts, NotificationPriority},
+        },
+        IncomingResponse, OutgoingRequest, SendAccessToken,
+    },
+    events::{room::power_levels::PowerLevelsEventContent, AnySyncRoomEvent, EventType},
+    identifiers::RoomName,
+    push::{Action, PushConditionRoomCtx, PushFormat, Ruleset, Tweak},
+    serde::Raw,
+    uint, RoomId, UInt, UserId,
+};
+use tracing::{error, info, warn};
+
+use std::{convert::TryFrom, fmt::Debug, mem, sync::Arc};
+
+use super::abstraction::Tree;
+
+pub struct PushData {
+    /// UserId + pushkey -> Pusher
+    pub(super) senderkey_pusher: Arc<dyn Tree>,
+}
+
+impl PushData {
+    #[tracing::instrument(skip(self, sender, pusher))]
+    pub fn set_pusher(&self, sender: &UserId, pusher: set_pusher::Pusher) -> Result<()> {
+        let mut key = sender.as_bytes().to_vec();
+        key.push(0xff);
+        key.extend_from_slice(pusher.pushkey.as_bytes());
+
+        // There are 2 kinds of pushers but the spec says: null deletes the pusher.
+        if pusher.kind.is_none() {
+            return self
+                .senderkey_pusher
+                .remove(&key)
+                .map(|_| ())
+                .map_err(Into::into);
+        }
+
+        self.senderkey_pusher.insert(
+            &key,
+            &serde_json::to_vec(&pusher).expect("Pusher is valid JSON value"),
+        )?;
+
+        Ok(())
+    }
+
+    #[tracing::instrument(skip(self, senderkey))]
+    pub fn get_pusher(&self, senderkey: &[u8]) -> Result<Option<get_pushers::Pusher>> {
+        self.senderkey_pusher
+            .get(senderkey)?
+            .map(|push| {
+                serde_json::from_slice(&*push)
+                    .map_err(|_| Error::bad_database("Invalid Pusher in db."))
+            })
+            .transpose()
+    }
+
+    #[tracing::instrument(skip(self, sender))]
+    pub fn get_pushers(&self, sender: &UserId) -> Result<Vec<get_pushers::Pusher>> {
+        let mut prefix = sender.as_bytes().to_vec();
+        prefix.push(0xff);
+
+        self.senderkey_pusher
+            .scan_prefix(prefix)
+            .map(|(_, push)| {
+                serde_json::from_slice(&*push)
+                    .map_err(|_| Error::bad_database("Invalid Pusher in db."))
+            })
+            .collect()
+    }
+
+    #[tracing::instrument(skip(self, sender))]
+    pub fn get_pusher_senderkeys<'a>(
+        &'a self,
+        sender: &UserId,
+    ) -> impl Iterator<Item = Vec<u8>> + 'a {
+        let mut prefix = sender.as_bytes().to_vec();
+        prefix.push(0xff);
+
+        self.senderkey_pusher.scan_prefix(prefix).map(|(k, _)| k)
+    }
+}
+
+#[tracing::instrument(skip(globals, destination, request))]
+pub async fn send_request<T: OutgoingRequest>(
+    globals: &crate::database::globals::Globals,
+    destination: &str,
+    request: T,
+) -> Result<T::IncomingResponse>
+where
+    T: Debug,
+{
+    let destination = destination.replace("/_matrix/push/v1/notify", "");
+
+    let http_request = request
+        .try_into_http_request::<BytesMut>(&destination, SendAccessToken::IfRequired(""))
+        .map_err(|e| {
+            warn!("Failed to find destination {}: {}", destination, e);
+            Error::BadServerResponse("Invalid destination")
+        })?
+        .map(|body| body.freeze());
+
+    let reqwest_request = reqwest::Request::try_from(http_request)
+        .expect("all http requests are valid reqwest requests");
+
+    // TODO: we could keep this very short and let expo backoff do it's thing...
+    //*reqwest_request.timeout_mut() = Some(Duration::from_secs(5));
+
+    let url = reqwest_request.url().clone();
+    let response = globals
+        .reqwest_client()?
+        .build()?
+        .execute(reqwest_request)
+        .await;
+
+    match response {
+        Ok(mut response) => {
+            // reqwest::Response -> http::Response conversion
+            let status = response.status();
+            let mut http_response_builder = http::Response::builder()
+                .status(status)
+                .version(response.version());
+            mem::swap(
+                response.headers_mut(),
+                http_response_builder
+                    .headers_mut()
+                    .expect("http::response::Builder is usable"),
+            );
+
+            let body = response.bytes().await.unwrap_or_else(|e| {
+                warn!("server error {}", e);
+                Vec::new().into()
+            }); // TODO: handle timeout
+
+            if status != 200 {
+                info!(
+                    "Push gateway returned bad response {} {}\n{}\n{:?}",
+                    destination,
+                    status,
+                    url,
+                    crate::utils::string_from_bytes(&body)
+                );
+            }
+
+            let response = T::IncomingResponse::try_from_http_response(
+                http_response_builder
+                    .body(body)
+                    .expect("reqwest body is valid http body"),
+            );
+            response.map_err(|_| {
+                info!(
+                    "Push gateway returned invalid response bytes {}\n{}",
+                    destination, url
+                );
+                Error::BadServerResponse("Push gateway returned bad response.")
+            })
+        }
+        Err(e) => Err(e.into()),
+    }
+}
+
+#[tracing::instrument(skip(user, unread, pusher, ruleset, pdu, db))]
+pub async fn send_push_notice(
+    user: &UserId,
+    unread: UInt,
+    pusher: &get_pushers::Pusher,
+    ruleset: Ruleset,
+    pdu: &PduEvent,
+    db: &Database,
+) -> Result<()> {
+    let mut notify = None;
+    let mut tweaks = Vec::new();
+
+    let power_levels: PowerLevelsEventContent = db
+        .rooms
+        .room_state_get(&pdu.room_id, &EventType::RoomPowerLevels, "")?
+        .map(|ev| {
+            serde_json::from_value(ev.content.clone())
+                .map_err(|_| Error::bad_database("invalid m.room.power_levels event"))
+        })
+        .transpose()?
+        .unwrap_or_default();
+
+    for action in get_actions(
+        user,
+        &ruleset,
+        &power_levels,
+        &pdu.to_sync_room_event(),
+        &pdu.room_id,
+        db,
+    )? {
+        let n = match action {
+            Action::DontNotify => false,
+            // TODO: Implement proper support for coalesce
+            Action::Notify | Action::Coalesce => true,
+            Action::SetTweak(tweak) => {
+                tweaks.push(tweak.clone());
+                continue;
+            }
+        };
+
+        if notify.is_some() {
+            return Err(Error::bad_database(
+                r#"Malformed pushrule contains more than one of these actions: ["dont_notify", "notify", "coalesce"]"#,
+            ));
+        }
+
+        notify = Some(n);
+    }
+
+    if notify == Some(true) {
+        send_notice(unread, pusher, tweaks, pdu, db).await?;
+    }
+    // Else the event triggered no actions
+
+    Ok(())
+}
+
+#[tracing::instrument(skip(user, ruleset, pdu, db))]
+pub fn get_actions<'a>(
+    user: &UserId,
+    ruleset: &'a Ruleset,
+    power_levels: &PowerLevelsEventContent,
+    pdu: &Raw<AnySyncRoomEvent>,
+    room_id: &RoomId,
+    db: &Database,
+) -> Result<&'a [Action]> {
+    let ctx = PushConditionRoomCtx {
+        room_id: room_id.clone(),
+        member_count: 10_u32.into(), // TODO: get member count efficiently
+        user_display_name: db
+            .users
+            .displayname(user)?
+            .unwrap_or_else(|| user.localpart().to_owned()),
+        users_power_levels: power_levels.users.clone(),
+        default_power_level: power_levels.users_default,
+        notification_power_levels: power_levels.notifications.clone(),
+    };
+
+    Ok(ruleset.get_actions(pdu, &ctx))
+}
+
+#[tracing::instrument(skip(unread, pusher, tweaks, event, db))]
+async fn send_notice(
+    unread: UInt,
+    pusher: &get_pushers::Pusher,
+    tweaks: Vec<Tweak>,
+    event: &PduEvent,
+    db: &Database,
+) -> Result<()> {
+    // TODO: email
+    if pusher.kind == PusherKind::Email {
+        return Ok(());
+    }
+
+    // TODO:
+    // Two problems with this
+    // 1. if "event_id_only" is the only format kind it seems we should never add more info
+    // 2. can pusher/devices have conflicting formats
+    let event_id_only = pusher.data.format == Some(PushFormat::EventIdOnly);
+    let url = if let Some(url) = &pusher.data.url {
+        url
+    } else {
+        error!("Http Pusher must have URL specified.");
+        return Ok(());
+    };
+
+    let mut device = Device::new(pusher.app_id.clone(), pusher.pushkey.clone());
+    let mut data_minus_url = pusher.data.clone();
+    // The url must be stripped off according to spec
+    data_minus_url.url = None;
+    device.data = Some(data_minus_url);
+
+    // Tweaks are only added if the format is NOT event_id_only
+    if !event_id_only {
+        device.tweaks = tweaks.clone();
+    }
+
+    let d = &[device];
+    let mut notifi = Notification::new(d);
+
+    notifi.prio = NotificationPriority::Low;
+    notifi.event_id = Some(&event.event_id);
+    notifi.room_id = Some(&event.room_id);
+    // TODO: missed calls
+    notifi.counts = NotificationCounts::new(unread, uint!(0));
+
+    if event.kind == EventType::RoomEncrypted
+        || tweaks
+            .iter()
+            .any(|t| matches!(t, Tweak::Highlight(true) | Tweak::Sound(_)))
+    {
+        notifi.prio = NotificationPriority::High
+    }
+
+    if event_id_only {
+        send_request(
+            &db.globals,
+            url,
+            send_event_notification::v1::Request::new(notifi),
+        )
+        .await?;
+    } else {
+        notifi.sender = Some(&event.sender);
+        notifi.event_type = Some(&event.kind);
+        let content = serde_json::value::to_raw_value(&event.content).ok();
+        notifi.content = content.as_deref();
+
+        if event.kind == EventType::RoomMember {
+            notifi.user_is_target = event.state_key.as_deref() == Some(event.sender.as_str());
+        }
+
+        let user_name = db.users.displayname(&event.sender)?;
+        notifi.sender_display_name = user_name.as_deref();
+        let room_name = db
+            .rooms
+            .room_state_get(&event.room_id, &EventType::RoomName, "")?
+            .map(|pdu| match pdu.content.get("name") {
+                Some(serde_json::Value::String(s)) => {
+                    Some(Box::<RoomName>::try_from(&**s).expect("room name is valid"))
+                }
+                _ => None,
+            })
+            .flatten();
+        notifi.room_name = room_name.as_deref();
+
+        send_request(
+            &db.globals,
+            url,
+            send_event_notification::v1::Request::new(notifi),
+        )
+        .await?;
+    }
+
+    // TODO: email
+
+    Ok(())
+}
@@ -1,28 +1,29 @@
-use crate::{utils, Error, Result};
+use crate::{database::abstraction::Tree, utils, Error, Result};
 use ruma::{
    events::{
        presence::{PresenceEvent, PresenceEventContent},
-        AnyEvent as EduEvent, SyncEphemeralRoomEvent,
+        AnyEphemeralRoomEvent, SyncEphemeralRoomEvent,
    },
    presence::PresenceState,
    serde::Raw,
+    signatures::CanonicalJsonObject,
    RoomId, UInt, UserId,
 };
 use std::{
-    collections::HashMap,
+    collections::{HashMap, HashSet},
    convert::{TryFrom, TryInto},
    mem,
+    sync::Arc,
 };

-#[derive(Clone)]
 pub struct RoomEdus {
-    pub(in super::super) readreceiptid_readreceipt: sled::Tree, // ReadReceiptId = RoomId + Count + UserId
-    pub(in super::super) roomuserid_privateread: sled::Tree, // RoomUserId = Room + User, PrivateRead = Count
-    pub(in super::super) roomuserid_lastprivatereadupdate: sled::Tree, // LastPrivateReadUpdate = Count
-    pub(in super::super) typingid_userid: sled::Tree, // TypingId = RoomId + TimeoutTime + Count
-    pub(in super::super) roomid_lasttypingupdate: sled::Tree, // LastRoomTypingUpdate = Count
-    pub(in super::super) presenceid_presence: sled::Tree, // PresenceId = RoomId + Count + UserId
-    pub(in super::super) userid_lastpresenceupdate: sled::Tree, // LastPresenceUpdate = Count
+    pub(in super::super) readreceiptid_readreceipt: Arc<dyn Tree>, // ReadReceiptId = RoomId + Count + UserId
+    pub(in super::super) roomuserid_privateread: Arc<dyn Tree>, // RoomUserId = Room + User, PrivateRead = Count
+    pub(in super::super) roomuserid_lastprivatereadupdate: Arc<dyn Tree>, // LastPrivateReadUpdate = Count
+    pub(in super::super) typingid_userid: Arc<dyn Tree>, // TypingId = RoomId + TimeoutTime + Count
+    pub(in super::super) roomid_lasttypingupdate: Arc<dyn Tree>, // LastRoomTypingUpdate = Count
+    pub(in super::super) presenceid_presence: Arc<dyn Tree>, // PresenceId = RoomId + Count + UserId
+    pub(in super::super) userid_lastpresenceupdate: Arc<dyn Tree>, // LastPresenceUpdate = Count
 }

 impl RoomEdus {
@@ -31,39 +32,39 @@ impl RoomEdus {
        &self,
        user_id: &UserId,
        room_id: &RoomId,
-        event: EduEvent,
+        event: AnyEphemeralRoomEvent,
        globals: &super::super::globals::Globals,
    ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
        prefix.push(0xff);

+        let mut last_possible_key = prefix.clone();
+        last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
+
        // Remove old entry
-        if let Some(old) = self
+        if let Some((old, _)) = self
            .readreceiptid_readreceipt
-            .scan_prefix(&prefix)
-            .keys()
-            .rev()
-            .filter_map(|r| r.ok())
-            .take_while(|key| key.starts_with(&prefix))
-            .find(|key| {
+            .iter_from(&last_possible_key, true)
+            .take_while(|(key, _)| key.starts_with(&prefix))
+            .find(|(key, _)| {
                key.rsplit(|&b| b == 0xff)
                    .next()
                    .expect("rsplit always returns an element")
-                    == user_id.to_string().as_bytes()
+                    == user_id.as_bytes()
            })
        {
            // This is the old room_latest
-            self.readreceiptid_readreceipt.remove(old)?;
+            self.readreceiptid_readreceipt.remove(&old)?;
        }

        let mut room_latest_id = prefix;
        room_latest_id.extend_from_slice(&globals.next_count()?.to_be_bytes());
        room_latest_id.push(0xff);
-        room_latest_id.extend_from_slice(&user_id.to_string().as_bytes());
+        room_latest_id.extend_from_slice(user_id.as_bytes());

        self.readreceiptid_readreceipt.insert(
-            room_latest_id,
-            &*serde_json::to_string(&event).expect("EduEvent::to_string always works"),
+            &room_latest_id,
+            &serde_json::to_vec(&event).expect("EduEvent::to_string always works"),
        )?;

        Ok(())
@@ -71,30 +72,51 @@ impl RoomEdus {

    /// Returns an iterator over the most recent read_receipts in a room that happened after the event with id `since`.
    #[tracing::instrument(skip(self))]
-    pub fn readreceipts_since(
-        &self,
+    pub fn readreceipts_since<'a>(
+        &'a self,
        room_id: &RoomId,
        since: u64,
-    ) -> Result<impl Iterator<Item = Result<Raw<ruma::events::AnySyncEphemeralRoomEvent>>>> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+    ) -> impl Iterator<Item = Result<(UserId, u64, Raw<ruma::events::AnySyncEphemeralRoomEvent>)>> + 'a
+    {
+        let mut prefix = room_id.as_bytes().to_vec();
        prefix.push(0xff);
+        let prefix2 = prefix.clone();

        let mut first_possible_edu = prefix.clone();
        first_possible_edu.extend_from_slice(&(since + 1).to_be_bytes()); // +1 so we don't send the event at since

-        Ok(self
-            .readreceiptid_readreceipt
-            .range(&*first_possible_edu..)
-            .filter_map(|r| r.ok())
-            .take_while(move |(k, _)| k.starts_with(&prefix))
-            .map(|(_, v)| {
-                Ok(serde_json::from_slice(&v).map_err(|_| {
-                    Error::bad_database("Read receipt in roomlatestid_roomlatest is invalid.")
-                })?)
-            }))
+        self.readreceiptid_readreceipt
+            .iter_from(&first_possible_edu, false)
+            .take_while(move |(k, _)| k.starts_with(&prefix2))
+            .map(move |(k, v)| {
+                let count =
+                    utils::u64_from_bytes(&k[prefix.len()..prefix.len() + mem::size_of::<u64>()])
+                        .map_err(|_| Error::bad_database("Invalid readreceiptid count in db."))?;
+                let user_id = UserId::try_from(
+                    utils::string_from_bytes(&k[prefix.len() + mem::size_of::<u64>() + 1..])
+                        .map_err(|_| {
+                            Error::bad_database("Invalid readreceiptid userid bytes in db.")
+                        })?,
+                )
+                .map_err(|_| Error::bad_database("Invalid readreceiptid userid in db."))?;
+
+                let mut json = serde_json::from_slice::<CanonicalJsonObject>(&v).map_err(|_| {
+                    Error::bad_database("Read receipt in roomlatestid_roomlatest is invalid json.")
+                })?;
+                json.remove("room_id");
+
+                Ok((
+                    user_id,
+                    count,
+                    Raw::from_json(
+                        serde_json::value::to_raw_value(&json).expect("json is valid raw value"),
+                    ),
+                ))
+            })
    }

    /// Sets a private read marker at `count`.
+    #[tracing::instrument(skip(self, globals))]
    pub fn private_read_set(
        &self,
        room_id: &RoomId,
@@ -102,9 +124,9 @@ impl RoomEdus {
        count: u64,
        globals: &super::super::globals::Globals,
    ) -> Result<()> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(user_id.as_bytes());

        self.roomuserid_privateread
            .insert(&key, &count.to_be_bytes())?;
@@ -118,22 +140,24 @@ impl RoomEdus {
    /// Returns the private read marker.
    #[tracing::instrument(skip(self))]
    pub fn private_read_get(&self, room_id: &RoomId, user_id: &UserId) -> Result<Option<u64>> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(user_id.as_bytes());

-        self.roomuserid_privateread.get(key)?.map_or(Ok(None), |v| {
-            Ok(Some(utils::u64_from_bytes(&v).map_err(|_| {
-                Error::bad_database("Invalid private read marker bytes")
-            })?))
-        })
+        self.roomuserid_privateread
+            .get(&key)?
+            .map_or(Ok(None), |v| {
+                Ok(Some(utils::u64_from_bytes(&v).map_err(|_| {
+                    Error::bad_database("Invalid private read marker bytes")
+                })?))
+            })
    }

    /// Returns the count of the last typing update in this room.
    pub fn last_privateread_update(&self, user_id: &UserId, room_id: &RoomId) -> Result<u64> {
-        let mut key = room_id.to_string().as_bytes().to_vec();
+        let mut key = room_id.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&user_id.to_string().as_bytes());
+        key.extend_from_slice(user_id.as_bytes());

        Ok(self
            .roomuserid_lastprivatereadupdate
@@ -155,7 +179,7 @@ impl RoomEdus {
        timeout: u64,
        globals: &super::super::globals::Globals,
    ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
        prefix.push(0xff);

        let count = globals.next_count()?.to_be_bytes();
@@ -166,10 +190,10 @@ impl RoomEdus {
        room_typing_id.extend_from_slice(&count);

        self.typingid_userid
-            .insert(&room_typing_id, &*user_id.to_string().as_bytes())?;
+            .insert(&room_typing_id, &*user_id.as_bytes())?;

        self.roomid_lasttypingupdate
-            .insert(&room_id.to_string().as_bytes(), &count)?;
+            .insert(room_id.as_bytes(), &count)?;

        Ok(())
    }
@@ -181,7 +205,7 @@ impl RoomEdus {
        room_id: &RoomId,
        globals: &super::super::globals::Globals,
    ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
        prefix.push(0xff);

        let user_id = user_id.to_string();
@@ -191,19 +215,16 @@ impl RoomEdus {
        // Maybe there are multiple ones from calling roomtyping_add multiple times
        for outdated_edu in self
            .typingid_userid
-            .scan_prefix(&prefix)
-            .filter_map(|r| r.ok())
-            .filter(|(_, v)| v == user_id.as_bytes())
+            .scan_prefix(prefix)
+            .filter(|(_, v)| &**v == user_id.as_bytes())
        {
-            self.typingid_userid.remove(outdated_edu.0)?;
+            self.typingid_userid.remove(&outdated_edu.0)?;
            found_outdated = true;
        }

        if found_outdated {
-            self.roomid_lasttypingupdate.insert(
-                &room_id.to_string().as_bytes(),
-                &globals.next_count()?.to_be_bytes(),
-            )?;
+            self.roomid_lasttypingupdate
+                .insert(room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
        }

        Ok(())
@@ -215,7 +236,7 @@ impl RoomEdus {
        room_id: &RoomId,
        globals: &super::super::globals::Globals,
    ) -> Result<()> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
        prefix.push(0xff);

        let current_timestamp = utils::millis_since_unix_epoch();
@@ -225,10 +246,8 @@ impl RoomEdus {
        // Find all outdated edus before inserting a new one
        for outdated_edu in self
            .typingid_userid
-            .scan_prefix(&prefix)
-            .keys()
-            .map(|key| {
-                let key = key?;
+            .scan_prefix(prefix)
+            .map(|(key, _)| {
                Ok::<_, Error>((
                    key.clone(),
                    utils::u64_from_bytes(
@@ -243,15 +262,13 @@ impl RoomEdus {
            .take_while(|&(_, timestamp)| timestamp < current_timestamp)
        {
            // This is an outdated edu (time > timestamp)
-            self.typingid_userid.remove(outdated_edu.0)?;
+            self.typingid_userid.remove(&outdated_edu.0)?;
            found_outdated = true;
        }

        if found_outdated {
-            self.roomid_lasttypingupdate.insert(
-                &room_id.to_string().as_bytes(),
-                &globals.next_count()?.to_be_bytes(),
-            )?;
+            self.roomid_lasttypingupdate
+                .insert(room_id.as_bytes(), &globals.next_count()?.to_be_bytes())?;
        }

        Ok(())
@@ -268,7 +285,7 @@ impl RoomEdus {

        Ok(self
            .roomid_lasttypingupdate
-            .get(&room_id.to_string().as_bytes())?
+            .get(room_id.as_bytes())?
            .map_or(Ok::<_, Error>(None), |bytes| {
                Ok(Some(utils::u64_from_bytes(&bytes).map_err(|_| {
                    Error::bad_database("Count in roomid_lastroomactiveupdate is invalid.")
@@ -281,29 +298,28 @@ impl RoomEdus {
        &self,
        room_id: &RoomId,
    ) -> Result<SyncEphemeralRoomEvent<ruma::events::typing::TypingEventContent>> {
-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
        prefix.push(0xff);

-        let mut user_ids = Vec::new();
+        let mut user_ids = HashSet::new();

        for user_id in self
            .typingid_userid
            .scan_prefix(prefix)
-            .values()
-            .map(|user_id| {
-                Ok::<_, Error>(
-                    UserId::try_from(utils::string_from_bytes(&user_id?).map_err(|_| {
-                        Error::bad_database("User ID in typingid_userid is invalid unicode.")
-                    })?)
-                    .map_err(|_| Error::bad_database("User ID in typingid_userid is invalid."))?,
-                )
+            .map(|(_, user_id)| {
+                UserId::try_from(utils::string_from_bytes(&user_id).map_err(|_| {
+                    Error::bad_database("User ID in typingid_userid is invalid unicode.")
+                })?)
+                .map_err(|_| Error::bad_database("User ID in typingid_userid is invalid."))
            })
        {
-            user_ids.push(user_id?);
+            user_ids.insert(user_id?);
        }

        Ok(SyncEphemeralRoomEvent {
-            content: ruma::events::typing::TypingEventContent { user_ids },
+            content: ruma::events::typing::TypingEventContent {
+                user_ids: user_ids.into_iter().collect(),
+            },
        })
    }

@@ -322,19 +338,19 @@ impl RoomEdus {

        let count = globals.next_count()?.to_be_bytes();

-        let mut presence_id = room_id.to_string().as_bytes().to_vec();
+        let mut presence_id = room_id.as_bytes().to_vec();
        presence_id.push(0xff);
        presence_id.extend_from_slice(&count);
        presence_id.push(0xff);
-        presence_id.extend_from_slice(&presence.sender.to_string().as_bytes());
+        presence_id.extend_from_slice(presence.sender.as_bytes());

        self.presenceid_presence.insert(
-            presence_id,
-            &*serde_json::to_string(&presence).expect("PresenceEvent can be serialized"),
+            &presence_id,
+            &serde_json::to_vec(&presence).expect("PresenceEvent can be serialized"),
        )?;

        self.userid_lastpresenceupdate.insert(
-            &user_id.to_string().as_bytes(),
+            user_id.as_bytes(),
            &utils::millis_since_unix_epoch().to_be_bytes(),
        )?;

@@ -345,7 +361,7 @@ impl RoomEdus {
    #[tracing::instrument(skip(self))]
    pub fn ping_presence(&self, user_id: &UserId) -> Result<()> {
        self.userid_lastpresenceupdate.insert(
-            &user_id.to_string().as_bytes(),
+            user_id.as_bytes(),
            &utils::millis_since_unix_epoch().to_be_bytes(),
        )?;

@@ -355,7 +371,7 @@ impl RoomEdus {
    /// Returns the timestamp of the last presence update of this user in millis since the unix epoch.
    pub fn last_presence_update(&self, user_id: &UserId) -> Result<Option<u64>> {
        self.userid_lastpresenceupdate
-            .get(&user_id.to_string().as_bytes())?
+            .get(user_id.as_bytes())?
            .map(|bytes| {
                utils::u64_from_bytes(&bytes).map_err(|_| {
                    Error::bad_database("Invalid timestamp in userid_lastpresenceupdate.")
@@ -364,8 +380,49 @@ impl RoomEdus {
            .transpose()
    }

+    pub fn get_last_presence_event(
+        &self,
+        user_id: &UserId,
+        room_id: &RoomId,
+    ) -> Result<Option<PresenceEvent>> {
+        let last_update = match self.last_presence_update(user_id)? {
+            Some(last) => last,
+            None => return Ok(None),
+        };
+
+        let mut presence_id = room_id.as_bytes().to_vec();
+        presence_id.push(0xff);
+        presence_id.extend_from_slice(&last_update.to_be_bytes());
+        presence_id.push(0xff);
+        presence_id.extend_from_slice(user_id.as_bytes());
+
+        self.presenceid_presence
+            .get(&presence_id)?
+            .map(|value| {
+                let mut presence = serde_json::from_slice::<PresenceEvent>(&value)
+                    .map_err(|_| Error::bad_database("Invalid presence event in db."))?;
+                let current_timestamp: UInt = utils::millis_since_unix_epoch()
+                    .try_into()
+                    .expect("time is valid");
+
+                if presence.content.presence == PresenceState::Online {
+                    // Don't set last_active_ago when the user is online
+                    presence.content.last_active_ago = None;
+                } else {
+                    // Convert from timestamp to duration
+                    presence.content.last_active_ago = presence
+                        .content
+                        .last_active_ago
+                        .map(|timestamp| current_timestamp - timestamp);
+                }
+
+                Ok(presence)
+            })
+            .transpose()
+    }
+
    /// Sets all users to offline who have been quiet for too long.
-    pub fn presence_maintain(
+    fn _presence_maintain(
        &self,
        rooms: &super::Rooms,
        globals: &super::super::globals::Globals,
@@ -375,7 +432,6 @@ impl RoomEdus {
        for (user_id_bytes, last_timestamp) in self
            .userid_lastpresenceupdate
            .iter()
-            .filter_map(|r| r.ok())
            .filter_map(|(k, bytes)| {
                Some((
                    k,
@@ -386,7 +442,7 @@ impl RoomEdus {
                        .ok()?,
                ))
            })
-            .take_while(|(_, timestamp)| current_timestamp - timestamp > 5 * 60_000)
+            .take_while(|(_, timestamp)| current_timestamp.saturating_sub(*timestamp) > 5 * 60_000)
        // 5 Minutes
        {
            // Send new presence events to set the user offline
@@ -398,15 +454,15 @@ impl RoomEdus {
                .try_into()
                .map_err(|_| Error::bad_database("Invalid UserId in userid_lastpresenceupdate."))?;
            for room_id in rooms.rooms_joined(&user_id).filter_map(|r| r.ok()) {
-                let mut presence_id = room_id.to_string().as_bytes().to_vec();
+                let mut presence_id = room_id.as_bytes().to_vec();
                presence_id.push(0xff);
                presence_id.extend_from_slice(&count);
                presence_id.push(0xff);
                presence_id.extend_from_slice(&user_id_bytes);

                self.presenceid_presence.insert(
-                    presence_id,
-                    &*serde_json::to_string(&PresenceEvent {
+                    &presence_id,
+                    &serde_json::to_vec(&PresenceEvent {
                        content: PresenceEventContent {
                            avatar_url: None,
                            currently_active: None,
@@ -424,7 +480,7 @@ impl RoomEdus {
            }

            self.userid_lastpresenceupdate.insert(
-                &user_id.to_string().as_bytes(),
+                user_id.as_bytes(),
                &utils::millis_since_unix_epoch().to_be_bytes(),
            )?;
        }
@@ -433,17 +489,17 @@ impl RoomEdus {
    }

    /// Returns an iterator over the most recent presence updates that happened after the event with id `since`.
-    #[tracing::instrument(skip(self, globals, rooms))]
+    #[tracing::instrument(skip(self, since, _rooms, _globals))]
    pub fn presence_since(
        &self,
        room_id: &RoomId,
        since: u64,
-        rooms: &super::Rooms,
-        globals: &super::super::globals::Globals,
+        _rooms: &super::Rooms,
+        _globals: &super::super::globals::Globals,
    ) -> Result<HashMap<UserId, PresenceEvent>> {
-        self.presence_maintain(rooms, globals)?;
+        //self.presence_maintain(rooms, globals)?;

-        let mut prefix = room_id.to_string().as_bytes().to_vec();
+        let mut prefix = room_id.as_bytes().to_vec();
        prefix.push(0xff);

        let mut first_possible_edu = prefix.clone();
@@ -452,8 +508,7 @@ impl RoomEdus {

        for (key, value) in self
            .presenceid_presence
-            .range(&*first_possible_edu..)
-            .filter_map(|r| r.ok())
+            .iter_from(&*first_possible_edu, false)
            .take_while(|(key, _)| key.starts_with(&prefix))
        {
            let user_id = UserId::try_from(
@@ -1,10 +1,12 @@
+use std::sync::Arc;
+
 use crate::Result;
 use ruma::{DeviceId, UserId};
-use sled::IVec;

-#[derive(Clone)]
+use super::abstraction::Tree;
+
 pub struct TransactionIds {
-    pub(super) userdevicetxnid_response: sled::Tree, // Response can be empty (/sendToDevice) or the event id (/send)
+    pub(super) userdevicetxnid_response: Arc<dyn Tree>, // Response can be empty (/sendToDevice) or the event id (/send)
 }

 impl TransactionIds {
@@ -21,7 +23,7 @@ impl TransactionIds {
        key.push(0xff);
        key.extend_from_slice(txn_id.as_bytes());

-        self.userdevicetxnid_response.insert(key, data)?;
+        self.userdevicetxnid_response.insert(&key, data)?;

        Ok(())
    }
@@ -31,7 +33,7 @@ impl TransactionIds {
        user_id: &UserId,
        device_id: Option<&DeviceId>,
        txn_id: &str,
-    ) -> Result<Option<IVec>> {
+    ) -> Result<Option<Vec<u8>>> {
        let mut key = user_id.as_bytes().to_vec();
        key.push(0xff);
        key.extend_from_slice(device_id.map(|d| d.as_bytes()).unwrap_or_default());
@@ -39,6 +41,6 @@ impl TransactionIds {
        key.extend_from_slice(txn_id.as_bytes());

        // If there's no entry, this is a new transaction
-        Ok(self.userdevicetxnid_response.get(key)?)
+        self.userdevicetxnid_response.get(&key)
    }
 }
@@ -1,15 +1,23 @@
-use crate::{Error, Result};
+use std::sync::Arc;
+
+use crate::{client_server::SESSION_ID_LENGTH, utils, Error, Result};
 use ruma::{
    api::client::{
        error::ErrorKind,
-        r0::uiaa::{IncomingAuthData, UiaaInfo},
+        r0::uiaa::{
+            IncomingAuthData, IncomingPassword, IncomingUserIdentifier::MatrixId, UiaaInfo,
+        },
    },
+    signatures::CanonicalJsonValue,
    DeviceId, UserId,
 };
+use tracing::error;
+
+use super::abstraction::Tree;

-#[derive(Clone)]
 pub struct Uiaa {
-    pub(super) userdeviceid_uiaainfo: sled::Tree, // User-interactive authentication
+    pub(super) userdevicesessionid_uiaainfo: Arc<dyn Tree>, // User-interactive authentication
+    pub(super) userdevicesessionid_uiaarequest: Arc<dyn Tree>, // UiaaRequest = canonical json value
 }

 impl Uiaa {
@@ -19,8 +27,20 @@ impl Uiaa {
        user_id: &UserId,
        device_id: &DeviceId,
        uiaainfo: &UiaaInfo,
+        json_body: &CanonicalJsonValue,
    ) -> Result<()> {
-        self.update_uiaa_session(user_id, device_id, Some(uiaainfo))
+        self.set_uiaa_request(
+            user_id,
+            device_id,
+            uiaainfo.session.as_ref().expect("session should be set"), // TODO: better session error handling (why is it optional in ruma?)
+            json_body,
+        )?;
+        self.update_uiaa_session(
+            user_id,
+            device_id,
+            uiaainfo.session.as_ref().expect("session should be set"),
+            Some(uiaainfo),
+        )
    }

    pub fn try_auth(
@@ -32,133 +52,161 @@ impl Uiaa {
        users: &super::users::Users,
        globals: &super::globals::Globals,
    ) -> Result<(bool, UiaaInfo)> {
-        if let IncomingAuthData::DirectRequest {
-            kind,
-            session,
-            auth_parameters,
-        } = &auth
-        {
-            let mut uiaainfo = session
-                .as_ref()
-                .map(|session| {
-                    Ok::<_, Error>(self.get_uiaa_session(&user_id, &device_id, session)?)
-                })
-                .unwrap_or_else(|| Ok(uiaainfo.clone()))?;
+        let mut uiaainfo = auth
+            .session()
+            .map(|session| self.get_uiaa_session(user_id, device_id, session))
+            .unwrap_or_else(|| Ok(uiaainfo.clone()))?;

+        if uiaainfo.session.is_none() {
+            uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
+        }
+
+        match auth {
            // Find out what the user completed
-            match &**kind {
-                "m.login.password" => {
-                    let identifier = auth_parameters.get("identifier").ok_or(Error::BadRequest(
-                        ErrorKind::MissingParam,
-                        "m.login.password needs identifier.",
-                    ))?;
-
-                    let identifier_type = identifier.get("type").ok_or(Error::BadRequest(
-                        ErrorKind::MissingParam,
-                        "Identifier needs a type.",
-                    ))?;
-
-                    if identifier_type != "m.id.user" {
+            IncomingAuthData::Password(IncomingPassword {
+                identifier,
+                password,
+                ..
+            }) => {
+                let username = match identifier {
+                    MatrixId(username) => username,
+                    _ => {
                        return Err(Error::BadRequest(
                            ErrorKind::Unrecognized,
                            "Identifier type not recognized.",
-                        ));
+                        ))
                    }
+                };

-                    let username = identifier
-                        .get("user")
-                        .ok_or(Error::BadRequest(
-                            ErrorKind::MissingParam,
-                            "Identifier needs user field.",
-                        ))?
-                        .as_str()
-                        .ok_or(Error::BadRequest(
-                            ErrorKind::BadJson,
-                            "User is not a string.",
-                        ))?;
-
-                    let user_id = UserId::parse_with_server_name(username, globals.server_name())
+                let user_id =
+                    UserId::parse_with_server_name(username.clone(), globals.server_name())
                        .map_err(|_| {
-                        Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid.")
-                    })?;
+                            Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid.")
+                        })?;

-                    let password = auth_parameters
-                        .get("password")
-                        .ok_or(Error::BadRequest(
-                            ErrorKind::MissingParam,
-                            "Password is missing.",
-                        ))?
-                        .as_str()
-                        .ok_or(Error::BadRequest(
-                            ErrorKind::BadJson,
-                            "Password is not a string.",
-                        ))?;
+                // Check if password is correct
+                if let Some(hash) = users.password_hash(&user_id)? {
+                    let hash_matches =
+                        argon2::verify_encoded(&hash, password.as_bytes()).unwrap_or(false);

-                    // Check if password is correct
-                    if let Some(hash) = users.password_hash(&user_id)? {
-                        let hash_matches =
-                            argon2::verify_encoded(&hash, password.as_bytes()).unwrap_or(false);
-
-                        if !hash_matches {
-                            uiaainfo.auth_error = Some(ruma::api::client::error::ErrorBody {
-                                kind: ErrorKind::Forbidden,
-                                message: "Invalid username or password.".to_owned(),
-                            });
-                            return Ok((false, uiaainfo));
-                        }
-                    }
-
-                    // Password was correct! Let's add it to `completed`
-                    uiaainfo.completed.push("m.login.password".to_owned());
-                }
-                "m.login.dummy" => {
-                    uiaainfo.completed.push("m.login.dummy".to_owned());
-                }
-                k => panic!("type not supported: {}", k),
-            }
-
-            // Check if a flow now succeeds
-            let mut completed = false;
-            'flows: for flow in &mut uiaainfo.flows {
-                for stage in &flow.stages {
-                    if !uiaainfo.completed.contains(stage) {
-                        continue 'flows;
+                    if !hash_matches {
+                        uiaainfo.auth_error = Some(ruma::api::client::error::ErrorBody {
+                            kind: ErrorKind::Forbidden,
+                            message: "Invalid username or password.".to_owned(),
+                        });
+                        return Ok((false, uiaainfo));
                    }
                }
-                // We didn't break, so this flow succeeded!
-                completed = true;
-            }

-            if !completed {
-                self.update_uiaa_session(user_id, device_id, Some(&uiaainfo))?;
-                return Ok((false, uiaainfo));
+                // Password was correct! Let's add it to `completed`
+                uiaainfo.completed.push("m.login.password".to_owned());
            }
-
-            // UIAA was successful! Remove this session and return true
-            self.update_uiaa_session(user_id, device_id, None)?;
-            Ok((true, uiaainfo))
-        } else {
-            panic!("FallbackAcknowledgement is not supported yet");
+            IncomingAuthData::Dummy(_) => {
+                uiaainfo.completed.push("m.login.dummy".to_owned());
+            }
+            k => error!("type not supported: {:?}", k),
        }
+
+        // Check if a flow now succeeds
+        let mut completed = false;
+        'flows: for flow in &mut uiaainfo.flows {
+            for stage in &flow.stages {
+                if !uiaainfo.completed.contains(stage) {
+                    continue 'flows;
+                }
+            }
+            // We didn't break, so this flow succeeded!
+            completed = true;
+        }
+
+        if !completed {
+            self.update_uiaa_session(
+                user_id,
+                device_id,
+                uiaainfo.session.as_ref().expect("session is always set"),
+                Some(&uiaainfo),
+            )?;
+            return Ok((false, uiaainfo));
+        }
+
+        // UIAA was successful! Remove this session and return true
+        self.update_uiaa_session(
+            user_id,
+            device_id,
+            uiaainfo.session.as_ref().expect("session is always set"),
+            None,
+        )?;
+        Ok((true, uiaainfo))
+    }
+
+    fn set_uiaa_request(
+        &self,
+        user_id: &UserId,
+        device_id: &DeviceId,
+        session: &str,
+        request: &CanonicalJsonValue,
+    ) -> Result<()> {
+        let mut userdevicesessionid = user_id.as_bytes().to_vec();
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(device_id.as_bytes());
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(session.as_bytes());
+
+        self.userdevicesessionid_uiaarequest.insert(
+            &userdevicesessionid,
+            &serde_json::to_vec(request).expect("json value to vec always works"),
+        )?;
+
+        Ok(())
+    }
+
+    pub fn get_uiaa_request(
+        &self,
+        user_id: &UserId,
+        device_id: &DeviceId,
+        session: &str,
+    ) -> Result<Option<CanonicalJsonValue>> {
+        let mut userdevicesessionid = user_id.as_bytes().to_vec();
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(device_id.as_bytes());
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(session.as_bytes());
+
+        self.userdevicesessionid_uiaarequest
+            .get(&userdevicesessionid)?
+            .map_or(Ok(None), |bytes| {
+                Ok::<_, Error>(Some(
+                    serde_json::from_str::<CanonicalJsonValue>(
+                        &utils::string_from_bytes(&bytes).map_err(|_| {
+                            Error::bad_database("Invalid uiaa request bytes in db.")
+                        })?,
+                    )
+                    .map_err(|_| Error::bad_database("Invalid uiaa request in db."))?,
+                ))
+            })
    }

    fn update_uiaa_session(
        &self,
        user_id: &UserId,
        device_id: &DeviceId,
+        session: &str,
        uiaainfo: Option<&UiaaInfo>,
    ) -> Result<()> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
-        userdeviceid.push(0xff);
-        userdeviceid.extend_from_slice(device_id.as_bytes());
+        let mut userdevicesessionid = user_id.as_bytes().to_vec();
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(device_id.as_bytes());
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(session.as_bytes());

        if let Some(uiaainfo) = uiaainfo {
-            self.userdeviceid_uiaainfo.insert(
-                &userdeviceid,
-                &*serde_json::to_string(&uiaainfo).expect("UiaaInfo::to_string always works"),
+            self.userdevicesessionid_uiaainfo.insert(
+                &userdevicesessionid,
+                &serde_json::to_vec(&uiaainfo).expect("UiaaInfo::to_vec always works"),
            )?;
        } else {
-            self.userdeviceid_uiaainfo.remove(&userdeviceid)?;
+            self.userdevicesessionid_uiaainfo
+                .remove(&userdevicesessionid)?;
        }

        Ok(())
@@ -170,14 +218,16 @@ impl Uiaa {
        device_id: &DeviceId,
        session: &str,
    ) -> Result<UiaaInfo> {
-        let mut userdeviceid = user_id.to_string().as_bytes().to_vec();
-        userdeviceid.push(0xff);
-        userdeviceid.extend_from_slice(device_id.as_bytes());
+        let mut userdevicesessionid = user_id.as_bytes().to_vec();
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(device_id.as_bytes());
+        userdevicesessionid.push(0xff);
+        userdevicesessionid.extend_from_slice(session.as_bytes());

        let uiaainfo = serde_json::from_slice::<UiaaInfo>(
            &self
-                .userdeviceid_uiaainfo
-                .get(&userdeviceid)?
+                .userdevicesessionid_uiaainfo
+                .get(&userdevicesessionid)?
                .ok_or(Error::BadRequest(
                    ErrorKind::Forbidden,
                    "UIAA session does not exist.",
@@ -185,18 +235,6 @@ impl Uiaa {
        )
        .map_err(|_| Error::bad_database("UiaaInfo in userdeviceid_uiaainfo is invalid."))?;

-        if uiaainfo
-            .session
-            .as_ref()
-            .filter(|&s| s == session)
-            .is_none()
-        {
-            return Err(Error::BadRequest(
-                ErrorKind::Forbidden,
-                "UIAA session token invalid.",
-            ));
-        }
-
        Ok(uiaainfo)
    }
 }
@@ -1,17 +1,12 @@
-use std::{
-    collections::HashMap,
-    sync::RwLock,
-    time::{Duration, Instant},
-};
-
-use log::error;
 use ruma::{
-    api::client::{error::ErrorKind, r0::uiaa::UiaaInfo},
-    events::room::message,
+    api::client::{
+        error::{Error as RumaError, ErrorKind},
+        r0::uiaa::UiaaInfo,
+    },
+    ServerName,
 };
 use thiserror::Error;
-
-use crate::{database::admin::AdminCommand, Database};
+use tracing::warn;

 #[cfg(feature = "conduit_bin")]
 use {
@@ -21,18 +16,29 @@ use {
        response::{self, Responder},
        Request,
    },
-    ruma::api::client::{error::Error as RumaError, r0::uiaa::UiaaResponse},
+    ruma::api::client::r0::uiaa::UiaaResponse,
+    tracing::error,
 };

 pub type Result<T> = std::result::Result<T, Error>;

 #[derive(Error, Debug)]
 pub enum Error {
-    #[error("There was a problem with the connection to the database.")]
+    #[cfg(feature = "sled")]
+    #[error("There was a problem with the connection to the sled database.")]
    SledError {
        #[from]
        source: sled::Error,
    },
+    #[cfg(feature = "sqlite")]
+    #[error("There was a problem with the connection to the sqlite database: {source}")]
+    SqliteError {
+        #[from]
+        source: rusqlite::Error,
+    },
+    #[cfg(feature = "heed")]
+    #[error("There was a problem with the connection to the heed database: {error}")]
+    HeedError { error: String },
    #[error("Could not generate an image.")]
    ImageError {
        #[from]
@@ -44,6 +50,13 @@ pub enum Error {
        source: reqwest::Error,
    },
    #[error("{0}")]
+    FederationError(Box<ServerName>, RumaError),
+    #[error("Could not do this io: {source}")]
+    IoError {
+        #[from]
+        source: std::io::Error,
+    },
+    #[error("{0}")]
    BadServerResponse(&'static str),
    #[error("{0}")]
    BadConfig(&'static str),
@@ -52,7 +65,6 @@ pub enum Error {
    BadDatabase(&'static str),
    #[error("uiaa")]
    Uiaa(UiaaInfo),
-
    #[error("{0}: {1}")]
    BadRequest(ErrorKind, &'static str),
    #[error("{0}")]
@@ -71,14 +83,16 @@ impl Error {
    }
 }

-#[cfg(feature = "conduit_bin")]
-impl<'r, 'o> Responder<'r, 'o> for Error
-where
-    'o: 'r,
-{
-    fn respond_to(self, r: &'r Request<'_>) -> response::Result<'o> {
-        if let Self::Uiaa(uiaainfo) = &self {
-            return RumaResponse::from(UiaaResponse::AuthResponse(uiaainfo.clone())).respond_to(r);
+impl Error {
+    pub fn to_response(&self) -> RumaResponse<UiaaResponse> {
+        if let Self::Uiaa(uiaainfo) = self {
+            return RumaResponse(UiaaResponse::AuthResponse(uiaainfo.clone()));
+        }
+
+        if let Self::FederationError(origin, error) = self {
+            let mut error = error.clone();
+            error.message = format!("Answer from {}: {}", origin, error.message);
+            return RumaResponse(UiaaResponse::MatrixError(error));
        }

        let message = format!("{}", self);
@@ -103,71 +117,22 @@ where
            _ => (Unknown, StatusCode::INTERNAL_SERVER_ERROR),
        };

-        RumaResponse::from(RumaError {
+        warn!("{}: {}", status_code, message);
+
+        RumaResponse(UiaaResponse::MatrixError(RumaError {
            kind,
            message,
            status_code,
-        })
-        .respond_to(r)
+        }))
    }
 }

-pub struct ConduitLogger {
-    pub db: Database,
-    pub last_logs: RwLock<HashMap<String, Instant>>,
-}
-
-impl log::Log for ConduitLogger {
-    fn enabled(&self, _metadata: &log::Metadata<'_>) -> bool {
-        true
+#[cfg(feature = "conduit_bin")]
+impl<'r, 'o> Responder<'r, 'o> for Error
+where
+    'o: 'r,
+{
+    fn respond_to(self, r: &'r Request<'_>) -> response::Result<'o> {
+        self.to_response().respond_to(r)
    }
-
-    fn log(&self, record: &log::Record<'_>) {
-        let output = format!("{} - {}", record.level(), record.args());
-
-        let match_mod_path =
-            |path: &str| path.starts_with("conduit::") || path.starts_with("state");
-
-        if self.enabled(record.metadata())
-            && (record.module_path().map_or(false, match_mod_path)
-                || record
-                    .module_path()
-                    .map_or(true, |path| !path.starts_with("rocket::")) // Rockets logs are annoying
-                    && record.metadata().level() <= log::Level::Warn)
-        {
-            let first_line = output
-                .lines()
-                .next()
-                .expect("lines always returns one item");
-
-            eprintln!("{}", output);
-
-            let mute_duration = match record.metadata().level() {
-                log::Level::Error => Duration::from_secs(60 * 5), // 5 minutes
-                log::Level::Warn => Duration::from_secs(60 * 60 * 24), // A day
-                _ => Duration::from_secs(60 * 60 * 24 * 7),       // A week
-            };
-
-            if self
-                .last_logs
-                .read()
-                .unwrap()
-                .get(first_line)
-                .map_or(false, |i| i.elapsed() < mute_duration)
-            // Don't post this log again for some time
-            {
-                return;
-            }
-
-            if let Ok(mut_last_logs) = &mut self.last_logs.try_write() {
-                mut_last_logs.insert(first_line.to_owned(), Instant::now());
-            }
-
-            self.db.admin.send(AdminCommand::SendMessage(
-                message::MessageEventContent::notice_plain(output),
-            ));
-        }
-    }
-
-    fn flush(&self) {}
 }
@@ -1,17 +1,19 @@
+#![allow(clippy::suspicious_else_formatting)]
+#![deny(clippy::dbg_macro)]
+
 pub mod appservice_server;
 pub mod client_server;
 mod database;
 mod error;
 mod pdu;
-mod push_rules;
 mod ruma_wrapper;
 pub mod server_server;
 mod utils;

-pub use database::Database;
+pub use database::{Config, Database};
 pub use error::{Error, Result};
 pub use pdu::PduEvent;
-pub use rocket::Config;
+pub use rocket::Config as RocketConfig;
 pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse};
 use std::ops::Deref;

@@ -1,4 +1,6 @@
 #![warn(rust_2018_idioms)]
+#![allow(clippy::suspicious_else_formatting)]
+#![deny(clippy::dbg_macro)]

 pub mod appservice_server;
 pub mod client_server;
@@ -7,13 +9,15 @@ pub mod server_server;
 mod database;
 mod error;
 mod pdu;
-mod push_rules;
 mod ruma_wrapper;
 mod utils;

+use std::sync::Arc;
+
 use database::Config;
 pub use database::Database;
 pub use error::{Error, Result};
+use opentelemetry::trace::{FutureExt, Tracer};
 pub use pdu::PduEvent;
 pub use rocket::State;
 use ruma::api::client::error::ErrorKind;
@@ -21,36 +25,18 @@ pub use ruma_wrapper::{ConduitResult, Ruma, RumaResponse};

 use rocket::{
    catch, catchers,
-    fairing::AdHoc,
    figment::{
        providers::{Env, Format, Toml},
        Figment,
    },
    routes, Request,
 };
-use tracing::span;
-use tracing_subscriber::{prelude::*, Registry};
+use tokio::sync::RwLock;
+use tracing_subscriber::{prelude::*, EnvFilter};

-fn setup_rocket() -> (rocket::Rocket, Config) {
-    // Force log level off, so we can use our own logger
-    std::env::set_var("CONDUIT_LOG_LEVEL", "off");
-
-    let config =
-        Figment::from(rocket::Config::release_default())
-            .merge(
-                Toml::file(Env::var("CONDUIT_CONFIG").expect(
-                    "The CONDUIT_CONFIG env var needs to be set. Example: /etc/conduit.toml",
-                ))
-                .nested(),
-            )
-            .merge(Env::prefixed("CONDUIT_").global());
-
-    let parsed_config = config
-        .extract::<Config>()
-        .expect("It looks like your config is invalid. Please take a look at the error");
-    let parsed_config2 = parsed_config.clone();
-
-    let rocket = rocket::custom(config)
+fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<rocket::Build> {
+    rocket::custom(config)
+        .manage(data)
        .mount(
            "/",
            routes![
@@ -64,6 +50,7 @@ fn setup_rocket() -> (rocket::Rocket, Config) {
                client_server::logout_all_route,
                client_server::change_password_route,
                client_server::deactivate_route,
+                client_server::third_party_route,
                client_server::get_capabilities_route,
                client_server::get_pushrules_all_route,
                client_server::set_pushrule_route,
@@ -74,16 +61,20 @@ fn setup_rocket() -> (rocket::Rocket, Config) {
                client_server::set_pushrule_actions_route,
                client_server::delete_pushrule_route,
                client_server::get_room_event_route,
+                client_server::get_room_aliases_route,
                client_server::get_filter_route,
                client_server::create_filter_route,
                client_server::set_global_account_data_route,
+                client_server::set_room_account_data_route,
                client_server::get_global_account_data_route,
+                client_server::get_room_account_data_route,
                client_server::set_displayname_route,
                client_server::get_displayname_route,
                client_server::set_avatar_url_route,
                client_server::get_avatar_url_route,
                client_server::get_profile_route,
                client_server::set_presence_route,
+                client_server::get_presence_route,
                client_server::upload_keys_route,
                client_server::get_keys_route,
                client_server::claim_keys_route,
@@ -155,6 +146,7 @@ fn setup_rocket() -> (rocket::Rocket, Config) {
                client_server::get_key_changes_route,
                client_server::get_pushers_route,
                client_server::set_pushers_route,
+                // client_server::third_party_route,
                client_server::upgrade_room_route,
                server_server::get_server_version_route,
                server_server::get_server_keys_route,
@@ -162,54 +154,119 @@ fn setup_rocket() -> (rocket::Rocket, Config) {
                server_server::get_public_rooms_route,
                server_server::get_public_rooms_filtered_route,
                server_server::send_transaction_message_route,
+                server_server::get_event_route,
                server_server::get_missing_events_route,
+                server_server::get_event_authorization_route,
+                server_server::get_room_state_route,
+                server_server::get_room_state_ids_route,
+                server_server::create_join_event_template_route,
+                server_server::create_join_event_v1_route,
+                server_server::create_join_event_v2_route,
+                server_server::create_invite_route,
+                server_server::get_devices_route,
+                server_server::get_room_information_route,
                server_server::get_profile_information_route,
+                server_server::get_keys_route,
+                server_server::claim_keys_route,
+            ],
+        )
+        .register(
+            "/",
+            catchers![
+                not_found_catcher,
+                forbidden_catcher,
+                unknown_token_catcher,
+                missing_token_catcher,
+                bad_json_catcher
            ],
        )
-        .register(catchers![
-            not_found_catcher,
-            forbidden_catcher,
-            unknown_token_catcher,
-            missing_token_catcher,
-            bad_json_catcher
-        ])
-        .attach(AdHoc::on_attach("Config", |rocket| async {
-            let data = Database::load_or_create(parsed_config2)
-                .await
-                .expect("config is valid");
-
-            data.sending
-                .start_handler(&data.globals, &data.rooms, &data.appservice);
-
-            Ok(rocket.manage(data))
-        }));
-
-    (rocket, parsed_config)
 }

 #[rocket::main]
 async fn main() {
-    let (rocket, config) = setup_rocket();
+    // Force log level off, so we can use our own logger
+    std::env::set_var("CONDUIT_LOG_LEVEL", "off");
+
+    let raw_config =
+        Figment::from(default_config())
+            .merge(
+                Toml::file(Env::var("CONDUIT_CONFIG").expect(
+                    "The CONDUIT_CONFIG env var needs to be set. Example: /etc/conduit.toml",
+                ))
+                .nested(),
+            )
+            .merge(Env::prefixed("CONDUIT_").global());
+
+    std::env::set_var("RUST_LOG", "warn");
+
+    let config = match raw_config.extract::<Config>() {
+        Ok(s) => s,
+        Err(e) => {
+            eprintln!("It looks like your config is invalid. The following error occured while parsing it: {}", e);
+            std::process::exit(1);
+        }
+    };
+
+    let start = async {
+        config.warn_deprecated();
+
+        let db = match Database::load_or_create(&config).await {
+            Ok(db) => db,
+            Err(e) => {
+                eprintln!(
+                    "The database couldn't be loaded or created. The following error occured: {}",
+                    e
+                );
+                std::process::exit(1);
+            }
+        };
+
+        let rocket = setup_rocket(raw_config, Arc::clone(&db))
+            .ignite()
+            .await
+            .unwrap();
+
+        Database::start_on_shutdown_tasks(db, rocket.shutdown()).await;
+
+        rocket.launch().await.unwrap();
+    };

    if config.allow_jaeger {
-        let (tracer, _uninstall) = opentelemetry_jaeger::new_pipeline()
-            .with_service_name("conduit")
-            .install()
+        opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
+        let tracer = opentelemetry_jaeger::new_pipeline()
+            .install_batch(opentelemetry::runtime::Tokio)
            .unwrap();
-        let telemetry = tracing_opentelemetry::layer().with_tracer(tracer);
-        Registry::default().with(telemetry).try_init().unwrap();

-        let root = span!(tracing::Level::INFO, "app_start", work_units = 2);
-        let _enter = root.enter();
+        let span = tracer.start("conduit");
+        start.with_current_context().await;
+        drop(span);

-        rocket.launch().await.unwrap();
+        println!("exporting");
+        opentelemetry::global::shutdown_tracer_provider();
    } else {
-        pretty_env_logger::init();
+        std::env::set_var("RUST_LOG", &config.log);

-        let root = span!(tracing::Level::INFO, "app_start", work_units = 2);
-        let _enter = root.enter();
+        let registry = tracing_subscriber::Registry::default();
+        if config.tracing_flame {
+            let (flame_layer, _guard) =
+                tracing_flame::FlameLayer::with_file("./tracing.folded").unwrap();
+            let flame_layer = flame_layer.with_empty_samples(false);

-        rocket.launch().await.unwrap();
+            let filter_layer = EnvFilter::new("trace,h2=off");
+
+            let subscriber = registry.with(filter_layer).with(flame_layer);
+            tracing::subscriber::set_global_default(subscriber).unwrap();
+            start.await;
+        } else {
+            let fmt_layer = tracing_subscriber::fmt::Layer::new();
+            let filter_layer = EnvFilter::try_from_default_env()
+                .or_else(|_| EnvFilter::try_new("info"))
+                .unwrap();
+
+            let subscriber = registry.with(filter_layer).with(fmt_layer);
+            tracing::subscriber::set_global_default(subscriber).unwrap();
+            start.await;
+        }
    }
 }

@@ -240,3 +297,30 @@ fn missing_token_catcher() -> Result<()> {
 fn bad_json_catcher() -> Result<()> {
    Err(Error::BadRequest(ErrorKind::BadJson, "Bad json."))
 }
+
+fn default_config() -> rocket::Config {
+    let mut config = rocket::Config::release_default();
+
+    {
+        let mut shutdown = &mut config.shutdown;
+
+        #[cfg(unix)]
+        {
+            use rocket::config::Sig;
+
+            shutdown.signals.insert(Sig::Term);
+            shutdown.signals.insert(Sig::Int);
+        }
+
+        // Once shutdown is triggered, this is the amount of seconds before rocket
+        // will forcefully start shutting down connections, this gives enough time to /sync
+        // requests and the like (which havent gotten the memo, somehow) to still complete gracefully.
+        shutdown.grace = 35;
+
+        // After the grace period, rocket starts shutting down connections, and waits at least this
+        // many seconds before forcefully shutting all of them down.
+        shutdown.mercy = 10;
+    }
+
+    config
+}
--- a/Show More
+++ b/Show More
@@ -1 +1 @@
 .47.0
 .52