Merge branch 'adjust-ci-for-next-and-master-development' into 'master'

CI: Adjust CI for master and next branch development model (master)

See merge request famedly/conduit!187

.gitlab-ci.yml

@@ -9,85 +9,6 @@ variables:
   FF_USE_FASTZIP: 1
   CACHE_COMPRESSION_LEVEL: fastest
 
-test:cargo:
-  stage: "test"
-  needs: []
-  image: "rust:latest"
-  tags: ["docker"]
-  variables:
-    CARGO_HOME: "cargohome"
-  cache:
-    paths:
-      - target
-      - cargohome
-    key: test_cache
-  interruptible: true
-  before_script:
-    - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
-    - apt-get update -yqq
-    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
-    - rustup component add clippy rustfmt
-  script:
-    - rustc --version && cargo --version  # Print version info for debugging
-    - cargo fmt --all -- --check
-    - cargo test --workspace --verbose --locked
-    - cargo clippy
-
-test:sytest:
-  stage: "test"
-  allow_failure: true
-  needs:
-    - "build:cargo:x86_64-unknown-linux-musl"
-  image:
-    name: "valkum/sytest-conduit:latest"
-    entrypoint: [""]
-  tags: ["docker"]
-  variables:
-    PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
-  before_script:
-    - "mkdir -p /app"
-    - "cp ./conduit-x86_64-unknown-linux-musl /app/conduit"
-    - "chmod +x /app/conduit"
-    - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
-    - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
-    - "cd /"
-  script:
-    - "SYTEST_EXIT_CODE=0"
-    - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
-    - "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap"
-    - "exit $SYTEST_EXIT_CODE"
-  artifacts:
-    when: always
-    paths:
-      - "$CI_PROJECT_DIR/sytest.xml"
-      - "$CI_PROJECT_DIR/results.tap"
-    reports:
-      junit: "$CI_PROJECT_DIR/sytest.xml"
-
-
-test:register:element-web-stable:
-  stage: "test"
-  needs:
-    - "build:cargo:x86_64-unknown-linux-gnu"
-  image: "buildkite/puppeteer:latest"
-  tags: ["docker"]
-  interruptible: true
-  script:
-    - "CONDUIT_CONFIG=tests/test-config.toml ./conduit-x86_64-unknown-linux-gnu > conduit.log &"
-    - "cd tests/client-element-web/"
-    - "npm install puppeteer"
-    - "node test-element-web-registration.js \"https://app.element.io/\" \"http://localhost:6167\""
-    - "killall --regexp \"conduit\""
-    - "cd ../.."
-    - "cat conduit.log"
-  artifacts:
-    paths:
-      - "tests/client-element-web/*.png"
-      - "*.log"
-    expire_in: 1 week
-    when: always
-  retry: 1
-
 
 # --------------------------------------------------------------------- #
 #  Cargo: Compiling for different architectures                         #
@@ -97,7 +18,9 @@ test:register:element-web-stable:
   stage: "build"
   needs: []
   rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
+    - if: '$CI_COMMIT_TAG'
   interruptible: true
   image: "rust:latest"
   tags: ["docker"]
@@ -105,7 +28,10 @@ test:register:element-web-stable:
     paths:
       - cargohome
       - target/
-    key: "build_cache-$TARGET"
+    key: "build_cache-$TARGET-release"
+  variables:
+    CARGO_PROFILE_RELEASE_LTO=true
+    CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1
   before_script:
     - 'echo "Building for target $TARGET"'
     - 'mkdir -p cargohome && CARGOHOME="cargohome"'
@@ -115,26 +41,23 @@ test:register:element-web-stable:
     - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
     - "rustup target add $TARGET"
   script:
-    # Set some cargo tuning here, because targets overwrite the 'variables'
-    - "export CARGO_INCREMENTAL=true"
-    - "export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16"
-    - "export CARGO_PROFILE_RELEASE_LTO=thin"
     - time cargo build --target $TARGET --release
     - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
+  artifacts:
+    expire_in: never
 
-build:cargo:x86_64-unknown-linux-gnu:
+
+build:release:cargo:x86_64-unknown-linux-gnu:
   extends: .build-cargo-shared-settings
   variables:
     TARGET: "x86_64-unknown-linux-gnu"
-  rules:
-    - if: "$CI_COMMIT_BRANCH"
   artifacts:
     name: "conduit-x86_64-unknown-linux-gnu"
     paths:
       - "conduit-x86_64-unknown-linux-gnu"
-    expose_as: "Release binary x86_64-unknown-linux-gnu"
+    expose_as: "Conduit for x86_64-unknown-linux-gnu"
 
-build:cargo:armv7-unknown-linux-gnueabihf:
+build:release:cargo:armv7-unknown-linux-gnueabihf:
   extends: .build-cargo-shared-settings
   variables:
     TARGET: "armv7-unknown-linux-gnueabihf"
@@ -146,9 +69,9 @@ build:cargo:armv7-unknown-linux-gnueabihf:
     name: "conduit-armv7-unknown-linux-gnueabihf"
     paths:
       - "conduit-armv7-unknown-linux-gnueabihf"
-    expose_as: "Release binary armv7-unknown-linux-gnueabihf"
+    expose_as: "Conduit for armv7-unknown-linux-gnueabihf"
 
-build:cargo:aarch64-unknown-linux-gnu:
+build:release:cargo:aarch64-unknown-linux-gnu:
   extends: .build-cargo-shared-settings
   variables:
     TARGET: "aarch64-unknown-linux-gnu"
@@ -162,13 +85,11 @@ build:cargo:aarch64-unknown-linux-gnu:
     name: "conduit-aarch64-unknown-linux-gnu"
     paths:
       - "conduit-aarch64-unknown-linux-gnu"
-    expose_as: "Release binary aarch64-unknown-linux-gnu"
+    expose_as: "Conduit for aarch64-unknown-linux-gnu"
 
-build:cargo:x86_64-unknown-linux-musl:
+build:release:cargo:x86_64-unknown-linux-musl:
   extends: .build-cargo-shared-settings
   image: "rust:alpine"
-  rules:
-    - if: '$CI_COMMIT_BRANCH' # Always run
   variables:
     TARGET: "x86_64-unknown-linux-musl"
   before_script:
@@ -181,7 +102,50 @@ build:cargo:x86_64-unknown-linux-musl:
     name: "conduit-x86_64-unknown-linux-musl"
     paths:
       - "conduit-x86_64-unknown-linux-musl"
-    expose_as: "Release binary x86_64-unknown-linux-musl"
+    expose_as: "Conduit for x86_64-unknown-linux-musl"
+
+
+
+.cargo-debug-shared-settings:
+  extends: ".build-cargo-shared-settings"
+  rules:
+    - if: '$CI_COMMIT_BRANCH'
+    - if: '$CI_COMMIT_TAG'
+  cache:
+    key: "build_cache-$TARGET-debug"
+  script:
+    - "time cargo build --target $TARGET"
+    - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
+  artifacts:
+    expire_in: 4 weeks
+
+build:debug:cargo:x86_64-unknown-linux-gnu:
+  extends: ".cargo-debug-shared-settings"
+  variables:
+    TARGET: "x86_64-unknown-linux-gnu"
+  artifacts:
+    name: "conduit-debug-x86_64-unknown-linux-gnu"
+    paths:
+      - "conduit-debug-x86_64-unknown-linux-gnu"
+    expose_as: "Conduit DEBUG for x86_64-unknown-linux-gnu"
+
+build:debug:cargo:x86_64-unknown-linux-musl:
+  extends: ".cargo-debug-shared-settings"
+  image: "rust:alpine"
+  variables:
+    TARGET: "x86_64-unknown-linux-musl"
+  before_script:
+    - 'echo "Building for target $TARGET"'
+    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
+    - "rustup target add $TARGET"
+    - "apk add libc-dev"
+  artifacts:
+    name: "conduit-debug-x86_64-unknown-linux-musl"
+    paths:
+      - "conduit-debug-x86_64-unknown-linux-musl"
+    expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"
+
 
 
 # --------------------------------------------------------------------- #
@@ -191,9 +155,11 @@ build:cargo:x86_64-unknown-linux-musl:
 
 .build-cargo-deb-shared-settings:
   stage: "build"
-  needs: []
+  needs: [ ]
   rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
+    - if: '$CI_COMMIT_TAG'
   interruptible: true
   image: "rust:latest"
   tags: ["docker"]
@@ -227,11 +193,11 @@ build:cargo-deb:x86_64-unknown-linux-gnu:
     expose_as: "Debian Package x86_64"
 
 
-
 # --------------------------------------------------------------------- #
 #  Create and publish docker image                                      #
 # --------------------------------------------------------------------- #
 
+# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
 .docker-shared-settings:
   stage: "build docker image"
   needs: []
@@ -248,11 +214,10 @@ build:cargo-deb:x86_64-unknown-linux-gnu:
     - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'
 
 
-# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
+build:docker:next:
-build:docker:main:
   extends: .docker-shared-settings
   needs:
-    - "build:cargo:x86_64-unknown-linux-musl"
+    - "build:release:cargo:x86_64-unknown-linux-musl"
   script:
     - >
       /kaniko/executor
@@ -260,16 +225,143 @@ build:docker:main:
       --context $CI_PROJECT_DIR
       --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
       --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
-      --build-arg "GIT_REF=$CI_COMMIT_REF_NAME"
+      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
       --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
-      --destination "$CI_REGISTRY_IMAGE/conduit:latest"
+      --destination "$CI_REGISTRY_IMAGE/conduit:next"
-      --destination "$CI_REGISTRY_IMAGE/conduit:alpine"
+      --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine"
       --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
-      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
-      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine"
       --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
   rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_BRANCH == "next"'
+
+
+build:docker:master:
+  extends: .docker-shared-settings
+  needs:
+    - "build:release:cargo:x86_64-unknown-linux-musl"
+  script:
+    - >
+      /kaniko/executor
+      $KANIKO_CACHE_ARGS
+      --context $CI_PROJECT_DIR
+      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
+      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
+      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
+      --destination "$CI_REGISTRY_IMAGE/conduit:latest"
+      --destination "$CI_REGISTRY_IMAGE/conduit:latest-alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest-alpine"
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+
+build:docker:tags:
+  extends: .docker-shared-settings
+  needs:
+    - "build:release:cargo:x86_64-unknown-linux-musl"
+  script:
+    - >
+      /kaniko/executor
+      $KANIKO_CACHE_ARGS
+      --context $CI_PROJECT_DIR
+      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
+      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
+      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
+      --destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG"
+      --destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG-alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG-alpine"
+  rules:
+    - if: '$CI_COMMIT_TAG'
+
+
+
+# --------------------------------------------------------------------- #
+#  Run tests                                                            #
+# --------------------------------------------------------------------- #
+
+test:cargo:
+  stage: "test"
+  needs: [ ]
+  image: "rust:latest"
+  tags: [ "docker" ]
+  variables:
+    CARGO_HOME: "cargohome"
+  cache:
+    paths:
+      - target
+      - cargohome
+    key: test_cache
+  interruptible: true
+  before_script:
+    - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
+    - apt-get update -yqq
+    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
+    - rustup component add clippy rustfmt
+  script:
+    - rustc --version && cargo --version  # Print version info for debugging
+    - cargo fmt --all -- --check
+    - cargo test --workspace --verbose --locked
+    - cargo clippy
+
+test:sytest:
+  stage: "test"
+  allow_failure: true
+  needs:
+    - "build:debug:cargo:x86_64-unknown-linux-musl"
+  image:
+    name: "valkum/sytest-conduit:latest"
+    entrypoint: [ "" ]
+  tags: [ "docker" ]
+  variables:
+    PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
+  before_script:
+    - "mkdir -p /app"
+    - "cp ./conduit-debug-x86_64-unknown-linux-musl /app/conduit"
+    - "chmod +x /app/conduit"
+    - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
+    - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
+    - "cd /"
+  script:
+    - "SYTEST_EXIT_CODE=0"
+    - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
+    - "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap"
+    - "exit $SYTEST_EXIT_CODE"
+  artifacts:
+    when: always
+    paths:
+      - "$CI_PROJECT_DIR/sytest.xml"
+      - "$CI_PROJECT_DIR/results.tap"
+    reports:
+      junit: "$CI_PROJECT_DIR/sytest.xml"
+
+
+test:register:element-web-stable:
+  stage: "test"
+  needs:
+    - "build:debug:cargo:x86_64-unknown-linux-gnu"
+  image: "buildkite/puppeteer:latest"
+  tags: [ "docker" ]
+  interruptible: true
+  script:
+    - "CONDUIT_CONFIG=tests/test-config.toml ./conduit-debug-x86_64-unknown-linux-gnu > conduit.log &"
+    - "cd tests/client-element-web/"
+    - "npm install puppeteer"
+    - "node test-element-web-registration.js \"https://app.element.io/\" \"http://localhost:6167\""
+    - "killall --regexp \"conduit\""
+    - "cd ../.."
+    - "cat conduit.log"
+  artifacts:
+    paths:
+      - "tests/client-element-web/*.png"
+      - "*.log"
+    expire_in: 1 week
+    when: always
+  retry: 1
 
 
 # --------------------------------------------------------------------- #
@@ -279,13 +371,15 @@ build:docker:main:
 publish:package:
   stage: "upload artifacts"
   needs:
-    - "build:cargo:x86_64-unknown-linux-gnu"
+    - "build:release:cargo:x86_64-unknown-linux-gnu"
-    - "build:cargo:armv7-unknown-linux-gnueabihf"
+    - "build:release:cargo:armv7-unknown-linux-gnueabihf"
-    - "build:cargo:aarch64-unknown-linux-gnu"
+    - "build:release:cargo:aarch64-unknown-linux-gnu"
-    - "build:cargo:x86_64-unknown-linux-musl"
+    - "build:release:cargo:x86_64-unknown-linux-musl"
     - "build:cargo-deb:x86_64-unknown-linux-gnu"
   rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
+    - if: '$CI_COMMIT_TAG'
   image: curlimages/curl:latest
   tags: ["docker"]
   variables:

Cargo.toml

@@ -6,7 +6,7 @@ authors = ["timokoesters <timo@koesters.xyz>"]
 homepage = "https://conduit.rs"
 repository = "https://gitlab.com/famedly/conduit"
 readme = "README.md"
-version = "0.1.0"
+version = "0.2.0"
 edition = "2018"
 
 # See more keys and their definitions at https://doc.rust-lang.org/cargo/reference/manifest.html
@@ -18,28 +18,29 @@ edition = "2018"
 rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests
 
 # Used for matrix spec type definitions and helpers
-#ruma = { git = "https://github.com/ruma/ruma", rev = "eb19b0e08a901b87d11b3be0890ec788cc760492", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
-ruma = { git = "https://github.com/timokoesters/ruma", rev = "a2d93500e1dbc87e7032a3c74f3b2479a7f84e93", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+#ruma = { git = "https://github.com/ruma/ruma", rev = "f5ab038e22421ed338396ece977b6b2844772ced", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+#ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 
 # Used for long polling and federation sender, should be the same as rocket::tokio
-tokio = "1.8.2"
+tokio = "1.11.0"
 # Used for storing data permanently
 sled = { version = "0.34.6", features = ["compression", "no_metrics"], optional = true }
 #sled = { git = "https://github.com/spacejam/sled.git", rev = "e4640e0773595229f398438886f19bca6f7326a2", features = ["compression"] }
 
 # Used for the http request / response body type for Ruma endpoints used with reqwest
-bytes = "1.0.1"
+bytes = "1.1.0"
 # Used for rocket<->ruma conversions
 http = "0.2.4"
 # Used to find data directory for default db path
 directories = "3.0.2"
 # Used for ruma wrapper
-serde_json = { version = "1.0.64", features = ["raw_value"] }
+serde_json = { version = "1.0.67", features = ["raw_value"] }
 # Used for appservice registration files
-serde_yaml = "0.8.17"
+serde_yaml = "0.8.20"
 # Used for pdu definition
-serde = "1.0.126"
+serde = "1.0.130"
 # Used for secure identifiers
 rand = "0.8.4"
 # Used to hash passwords
@@ -49,9 +50,9 @@ reqwest = { version = "0.11.4", default-features = false, features = ["rustls-tl
 # Custom TLS verifier
 rustls = { version = "0.19.1", features = ["dangerous_configuration"] }
 rustls-native-certs = "0.5.0"
-webpki = "0.21.0"
+webpki = "0.22.0"
 # Used for conduit::Error type
-thiserror = "1.0.26"
+thiserror = "1.0.28"
 # Used to generate thumbnails for images
 image = { version = "0.23.14", default-features = false, features = ["jpeg", "png", "gif"] }
 # Used to encode server public key
@@ -66,19 +67,20 @@ regex = "1.5.4"
 jsonwebtoken = "7.2.0"
 # Performance measurements
 tracing = { version = "0.1.26", features = ["release_max_level_warn"] }
-opentelemetry = "0.15.0"
+tracing-subscriber = "0.2.20"
-tracing-subscriber = "0.2.19"
+tracing-opentelemetry = "0.15.0"
-tracing-opentelemetry = "0.14.0"
 tracing-flame = "0.1.0"
-opentelemetry-jaeger = "0.14.0"
+opentelemetry = { version = "0.16.0", features = ["rt-tokio"] }
+opentelemetry-jaeger = { version = "0.15.0", features = ["rt-tokio"] }
 pretty_env_logger = "0.4.0"
 lru-cache = "0.1.2"
 rusqlite = { version = "0.25.3", optional = true, features = ["bundled"] }
-parking_lot = { version = "0.11.1", optional = true }
+parking_lot = { version = "0.11.2", optional = true }
 crossbeam = { version = "0.8.1", optional = true }
 num_cpus = "1.13.0"
 threadpool = "1.8.1"
 heed = { git = "https://github.com/timokoesters/heed.git", rev = "f6f825da7fb2c758867e05ad973ef800a6fe1d5d", optional = true }
+thread_local = "1.1.3"
 
 [features]
 default = ["conduit_bin", "backend_sqlite"]
@@ -119,6 +121,20 @@ conf-files = [
 maintainer-scripts = "debian/"
 systemd-units = { unit-name = "matrix-conduit" }
 
-# For flamegraphs:
+[profile.dev]
-#[profile.release]
+lto = 'thin'
-#debug = true
+incremental = true
+
+[profile.release]
+lto = 'thin'
+incremental = true
+
+codegen-units=32
+# If you want to make flamegraphs, enable debug info:
+# debug = true
+
+# For releases also try to max optimizations for dependencies:
+[profile.release.build-override]
+opt-level = 3
+[profile.release.package."*"]
+opt-level = 3

DEPLOY.md

@@ -6,8 +6,7 @@ If you run into any problems while setting up Conduit, write an email to `timo@k
 
 ## Installing Conduit
 
-You may simply download the binary that fits your machine. Run `uname -m` to see
+You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:
-what you need. Now copy the right url:
 
 | CPU Architecture     | GNU (Ubuntu, Debian, ArchLinux, ...)  | MUSL (Alpine, ... )     |
 | -------------------- | ------------------------------------- | ----------------------- |
@@ -15,10 +14,13 @@ what you need. Now copy the right url:
 | armv7 (Raspberry Pi) | [Download][armv7-gnu]                 | -                       |
 | armv8 / aarch64      | [Download][armv8-gnu]                 | -                       |
 
-[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:cargo:x86_64-unknown-linux-gnu
+[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:release:cargo:x86_64-unknown-linux-gnu
-[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:cargo:x86_64-unknown-linux-musl
+
-[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:cargo:armv7-unknown-linux-gnueabihf
+[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
-[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:cargo:aarch64-unknown-linux-gnu
+
+[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:release:cargo:armv7-unknown-linux-gnueabihf
+
+[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:release:cargo:aarch64-unknown-linux-gnu
 
 ```bash
 $ sudo wget -O /usr/local/bin/matrix-conduit <url>
@@ -26,6 +28,7 @@ $ sudo chmod +x /usr/local/bin/matrix-conduit
 ```
 
 Alternatively, you may compile the binary yourself using
+
 ```bash
 $ cargo build --release
 ```
@@ -41,7 +44,7 @@ This also allows you to make sure that the file permissions are correctly set up
 
 In Debian you can use this command to create a Conduit user:
 
-```
+```bash
 sudo adduser --system conduit --no-create-home
 ```
 
@@ -104,8 +107,8 @@ port = 6167
 # Max size for uploads
 max_request_size = 20_000_000 # in bytes
 
-# Disabling registration means no new users will be able to register on this server
+# Enables registration. If set to false, no users can register on this server.
-allow_registration = false
+allow_registration = true
 
 # Disable encryption, so no new encrypted rooms can be created
 # Note: existing rooms will continue to work
@@ -128,13 +131,13 @@ address = "127.0.0.1" # This makes sure Conduit can only be reached using the re
 As we are using a Conduit specific user we need to allow it to read the config.
 To do that you can run this command on Debian:
 
-```
+```bash
 sudo chown -R conduit:nogroup /etc/matrix-conduit
 ```
 
 If you use the default database path you also need to run this:
 
-```
+```bash
 sudo mkdir -p /var/lib/matrix-conduit/conduit_db
 sudo chown -R conduit:nogroup /var/lib/matrix-conduit/conduit_db
 ```
@@ -148,7 +151,7 @@ This depends on whether you use Apache, Nginx or another web server.
 
 Create `/etc/apache2/sites-enabled/050-conduit.conf` and copy-and-paste this:
 
-```
+```apache
 Listen 8448
 
 <VirtualHost *:443 *:8448>
@@ -177,7 +180,7 @@ $ sudo systemctl reload apache2
 If you use Nginx and not Apache, add the following server section inside the
 http section of `/etc/nginx/nginx.conf`
 
-```
+```nginx
 server {
     listen 443 ssl http2;
     listen [::]:443 ssl http2;
@@ -228,4 +231,15 @@ Set it to start automatically when your system boots with:
 $ sudo systemctl enable conduit
 ```
 
+## How do I know it works?
+
+You can open <https://app.element.io>, enter your homeserver and try to register.
+
+You can also use these commands as a quick health check.
+
+```bash
+$ curl https://your.server.name/_matrix/client/versions
+$ curl https://your.server.name:8448/_matrix/client/versions
+```
+
 If you want to set up an appservice, take a look at the [Appservice Guide](APPSERVICES.md).

Dockerfile

@@ -7,25 +7,29 @@
 # Alpine build image to build Conduit's statically compiled binary
 FROM alpine:3.14 as builder
 
+# Install packages needed for building all crates
+RUN apk add --no-cache \
+        cargo \
+        openssl-dev
+
 # Specifies if the local project is build or if Conduit gets build
 # from the official git repository. Defaults to the git repo.
 ARG LOCAL=false
 # Specifies which revision/commit is build. Defaults to HEAD
 ARG GIT_REF=origin/master
 
-# Install packages needed for building all crates
-RUN apk add --no-cache \
-        cargo \
-        openssl-dev
-
-
 # Copy project files from current folder
 COPY . .
 # Build it from the copied local files or from the official git repository
 RUN if [[ $LOCAL == "true" ]]; then \
+        mv ./docker/healthcheck.sh . ; \
+        echo "Building from local source..." ; \
         cargo install --path . ; \
     else \
-        cargo install --git "https://gitlab.com/famedly/conduit.git" --rev ${GIT_REF}; \
+        echo "Building revision '${GIT_REF}' from online source..." ; \
+        cargo install --git "https://gitlab.com/famedly/conduit.git" --rev ${GIT_REF} ; \
+        echo "Loadings healthcheck script from online source..." ; \
+        wget "https://gitlab.com/famedly/conduit/-/raw/${GIT_REF#origin/}/docker/healthcheck.sh" ; \
     fi
 
 ########################## RUNTIME IMAGE ##########################
@@ -64,6 +68,7 @@ EXPOSE 6167
 # /srv/conduit and create data folder for database
 RUN mkdir -p /srv/conduit/.local/share/conduit
 COPY --from=builder /root/.cargo/bin/conduit /srv/conduit/
+COPY --from=builder ./healthcheck.sh /srv/conduit/
 
 # Add www-data user and group with UID 82, as used by alpine
 # https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
@@ -82,10 +87,7 @@ RUN apk add --no-cache \
         libgcc
 
 # Test if Conduit is still alive, uses the same endpoint as Element
-HEALTHCHECK --start-period=5s \
+HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh
-    CMD curl --fail -s "http://localhost:$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')/_matrix/client/versions" || \
-        curl -k --fail -s "https://localhost:$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')/_matrix/client/versions" || \
-        exit 1
 
 # Set user to www-data
 USER www-data

README.md

@@ -3,90 +3,42 @@
 
 #### What is the goal?
 
-A fast Matrix homeserver that's easy to set up and just works. You can install
+An efficient Matrix homeserver that's easy to set up and just works. You can install
 it on a mini-computer like the Raspberry Pi to host Matrix for your family,
 friends or company.
 
 
 #### Can I try it out?
 
-Yes! Just open a Matrix client (<https://app.element.io> or Element Android for
+Yes! You can test our Conduit instance by opening a Matrix client (<https://app.element.io> or Element Android for
-example) and register on the `https://conduit.koesters.xyz` homeserver.
+example) and registering on the `conduit.rs` homeserver.
 
-
+It is hosted on a ODROID HC 2 with 2GB RAM and a SAMSUNG Exynos 5422 CPU, which
-#### What is it built on?
+was used in the Samsung Galaxy S5. It joined many big rooms including Matrix
-
+HQ.
-- [Ruma](https://www.ruma.io): Useful structures for endpoint requests and
-  responses that can be (de)serialized
-- [Sled](https://github.com/spacejam/sled): A simple (key, value) database with
-  good performance
-- [Rocket](https://rocket.rs): A flexible web framework
 
 
 #### What is the current status?
 
-Conduit can already be used chat with other users on Conduit, chat with users
+As of 2021-09-01, Conduit is Beta, meaning you can join and participate in most
-from other Matrix servers and even to chat with users on other platforms using
+Matrix rooms, but not all features are supported and you might run into bugs
-appservices. When chatting with users on the same Conduit server, everything
+from time to time.
-should work assuming you use a compatible client.
-
-**You should not join Matrix rooms without asking the admins first.** We do not
-know whether Conduit is safe for general use yet, so you should assume there is
-some chance that it breaks rooms permanently for all participating users. We
-are not aware of such a bug today, but we would like to do more testing.
 
 There are still a few important features missing:
 
-- Database stability (currently you might have to do manual upgrades or even wipe the db for new versions)
+- E2EE verification over federation
-- Edge cases for end-to-end encryption over federation
+- Outgoing read receipts, typing, presence over federation
-- Typing and presence over federation
-- Lots of testing
 
 Check out the [Conduit 1.0 Release Milestone](https://gitlab.com/famedly/conduit/-/milestones/3).
 
 
 #### How can I deploy my own?
 
-##### Deploy
+Simple install (this was tested the most): [DEPLOY.md](DEPLOY.md)\
+Debian package: [debian/README.Debian](debian/README.Debian)\
+Docker: [docker/README.md](docker/README.md)
 
-Download or compile a Conduit binary, set up the config and call it from somewhere like a systemd script. [Read
+If you want to connect an Appservice to Conduit, take a look at [APPSERVICES.md](APPSERVICES.md).
-more](DEPLOY.md)
-
-If you want to connect an Appservice to Conduit, take a look at the [Appservice Guide](APPSERVICES.md).
-
-##### Deploy using a Debian package
-
-You need to have the `deb` helper command installed that creates Debian packages from Cargo projects (see [cargo-deb](https://github.com/mmstick/cargo-deb/) for more info):
-
-```shell
-$ cargo install cargo-deb
-```
-
-Then, you can create and install a Debian package at a whim:
-
-```shell
-$ cargo deb
-$ dpkg -i target/debian/matrix-conduit_0.1.0_amd64.deb
-```
-
-This will build, package, install, configure and start Conduit. [Read more](debian/README.Debian).
-
-Note that `cargo deb` supports [cross-compilation](https://github.com/mmstick/cargo-deb/#cross-compilation) too!
-Official Debian packages will follow once Conduit starts to have stable releases.
-
-##### Deploy using Docker
-
-Pull and run the docker image with
-
-``` bash
-docker pull matrixconduit/matrix-conduit:latest
-docker run -d -p 8448:8000 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
-```
-
-> <b>Note:</b> You also need to supply a `conduit.toml` config file, you can find an example [here](./conduit-example.toml).
-> Or you can pass in `-e CONDUIT_CONFIG=""` and configure Conduit purely with env vars.
-
-Or build and run it with docker or docker-compose. [Read more](docker/README.md)
 
 
 #### How can I contribute?
@@ -98,6 +50,17 @@ Or build and run it with docker or docker-compose. [Read more](docker/README.md)
 3. Fork the repo and work on the issue. #conduit:nordgedanken.dev is happy to help :)
 4. Submit a MR
 
+
+#### Thanks to
+
+Thanks to Famedly, Prototype Fund (DLR and German BMBF) and all other individuals for financially supporting this project.
+
+Thanks to the contributors to Conduit and all libraries we use, for example:
+
+- Ruma: A clean library for the Matrix Spec in Rust
+- Rocket: A flexible web framework
+
+
 #### Donate
 
 Liberapay: <https://liberapay.com/timokoesters/>\

conduit-example.toml

@@ -22,8 +22,8 @@ port = 6167
 # Max size for uploads
 max_request_size = 20_000_000 # in bytes
 
-# Disable registration. No new users will be able to register on this server
+# Enables registration. If set to false, no users can register on this server.
-#allow_registration = false
+allow_registration = true
 
 # Disable encryption, so no new encrypted rooms can be created
 # Note: existing rooms will continue to work

debian/postinst

@@ -62,8 +62,8 @@ port = ${CONDUIT_PORT}
 # Max size for uploads
 max_request_size = 20_000_000 # in bytes
 
-# Disable registration. No new users will be able to register on this server.
+# Enables registration. If set to false, no users can register on this server.
-#allow_registration = false
+allow_registration = true
 
 # Disable encryption, so no new encrypted rooms can be created.
 # Note: Existing rooms will continue to work.

docker-compose.yml

@@ -27,6 +27,7 @@ services:
         environment:
             CONDUIT_SERVER_NAME: localhost:6167  # replace with your own name
             CONDUIT_TRUSTED_SERVERS: '["matrix.org"]'
+            CONDUIT_ALLOW_REGISTRATION: 'true'
             ### Uncomment and change values as desired
             # CONDUIT_ADDRESS: 0.0.0.0
             # CONDUIT_PORT: 6167
@@ -34,7 +35,6 @@ services:
             # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging
             # CONDUIT_LOG: info  # default is: "info,rocket=off,_=off,sled=off"
             # CONDUIT_ALLOW_JAEGER: 'false'
-            # CONDUIT_ALLOW_REGISTRATION : 'false'
             # CONDUIT_ALLOW_ENCRYPTION: 'false'
             # CONDUIT_ALLOW_FEDERATION: 'false'
             # CONDUIT_DATABASE_PATH: /srv/conduit/.local/share/conduit

docker/README.md

@@ -40,18 +40,27 @@ which also will tag the resulting image as `matrixconduit/matrix-conduit:latest`
 After building the image you can simply run it with
 
 ``` bash
-docker run -d -p 8448:8000 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
+docker run -d -p 8448:6167 -v ~/conduit.toml:/srv/conduit/conduit.toml -v db:/srv/conduit/.local/share/conduit matrixconduit/matrix-conduit:latest
 ```
 
-For detached mode, you also need to use the `-d` flag. You also need to supply a `conduit.toml` config file, you can find an example [here](../conduit-example.toml).
+or you can skip the build step and pull the image from one of the following registries:
+
+| Registry        | Image                                                                                                               | Size                                                                                        |
+| --------------- | ------------------------------------------------------------------------------------------------------------------- | ------------------------------------------------------------------------------------------- |
+| Docker Hub      | [matrixconduit/matrix-conduit:latest](https://hub.docker.com/r/matrixconduit/matrix-conduit)                        | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) |
+| GitLab Registry | [registry.gitlab.com/famedly/conduit/conduit:latest](https://gitlab.com/famedly/conduit/container_registry/2134341) | ![Image Size](https://img.shields.io/docker/image-size/matrixconduit/matrix-conduit/latest) |
+
+The `-d` flag lets the container run in detached mode. You now need to supply a `conduit.toml` config file, an example can be found [here](../conduit-example.toml).
 You can pass in different env vars to change config values on the fly. You can even configure Conduit completely by using env vars, but for that you need
-too pass `-e CONDUIT_CONFIG=""` into your container. For an overview of possible values, please take a look at the `docker-compose.yml` file.
+to pass `-e CONDUIT_CONFIG=""` into your container. For an overview of possible values, please take a look at the `docker-compose.yml` file.
+
 If you just want to test Conduit for a short time, you can use the `--rm` flag, which will clean up everything related to your container after you stop it.
 
 
 ## Docker-compose
 
-If the docker command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, use the [`docker-compose.traefik.yml`](docker-compose.traefik.yml) including [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) or the normal [`docker-compose.yml`](../docker-compose.yml) for every other reverse proxy.
+If the docker command is not for you or your setup, you can also use one of the provided `docker-compose` files. Depending on your proxy setup, use the [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) for Traefik (don't forget to remove `.traefik` from the filenames) or the normal [`docker-compose.yml`](../docker-compose.yml) for every other reverse proxy. Additional info about deploying
+Conduit can be found [here](../DEPLOY.md).
 
 
 ### Build
@@ -67,8 +76,57 @@ This will also start the container right afterwards, so if want it to run in det
 
 ### Run
 
-If you already have built the image, you can just start the container and everything else in the compose file in detached mode with:
+If you already have built the image or want to use one from the registries, you can just start the container and everything else in the compose file in detached mode with:
 
 ``` bash
 docker-compose up -d
 ```
+
+> **Note:** Don't forget to modify and adjust the compose file to your needs.
+
+### Use Traefik as Proxy
+
+As a container user, you probably know about Traefik. It is a easy to use reverse proxy for making containerized app and services available through the web. With the
+two provided files, [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml), it is
+equally easy to deploy and use Conduit, with a little caveat. If you already took a look at the files, then you should have seen the `well-known` service, and that is
+the little caveat. Traefik is simply a proxy and loadbalancer and is not able to serve any kind of content, but for Conduit to federate, we need to either expose ports
+`443` and `8448` or serve two endpoints `.well-known/matrix/client` and `.well-known/matrix/server`.
+
+With the service `well-known` we use a single `nginx` container that will serve those two files.
+
+So...step by step:
+
+1. Copy [`docker-compose.traefik.yml`](docker-compose.traefik.yml) and [`docker-compose.override.traefik.yml`](docker-compose.override.traefik.yml) from the repository and remove `.traefik` from the filenames.
+2. Open both files and modify/adjust them to your needs. Meaning, change the `CONDUIT_SERVER_NAME` and the volume host mappings according to your needs.
+3. Create the `conduit.toml` config file, an example can be found [here](../conduit-example.toml), or set `CONDUIT_CONFIG=""` and configure Conduit per env vars.
+4. Uncomment the `element-web` service if you want to host your own Element Web Client and create a `element_config.json`.
+5. Create the files needed by the `well-known` service.
+    - `./nginx/matrix.conf` (relative to the compose file, you can change this, but then also need to change the volume mapping)
+      ```nginx
+      server {
+          server_name <SUBDOMAIN>.<DOMAIN>;
+          listen      80 default_server;
+
+          location /.well-known/matrix/ {
+              root /var/www;
+              default_type application/json;
+              add_header Access-Control-Allow-Origin *;
+          }
+      }
+      ```
+    - `./nginx/www/.well-known/matrix/client` (relative to the compose file, you can change this, but then also need to change the volume mapping)
+      ```json
+      {
+          "m.homeserver": {
+              "base_url": "https://<SUBDOMAIN>.<DOMAIN>"
+          }
+      }
+      ```
+    - `./nginx/www/.well-known/matrix/server` (relative to the compose file, you can change this, but then also need to change the volume mapping)
+      ```json
+      {
+          "m.server": "<SUBDOMAIN>.<DOMAIN>:443"
+      }
+      ```
+6. Run `docker-compose up -d`
+7. Connect to your homeserver with your preferred client and create a user. You should do this immediatly after starting Conduit, because the first created user is the admin.

docker/ci-binaries-packaging.Dockerfile

@@ -9,6 +9,12 @@
 
 FROM alpine:3.14
 
+# Install packages needed to run Conduit
+RUN apk add --no-cache \
+        ca-certificates \
+        curl \
+        libgcc
+
 ARG CREATED
 ARG VERSION
 ARG GIT_REF
@@ -36,6 +42,10 @@ EXPOSE 6167
 # create data folder for database
 RUN mkdir -p /srv/conduit/.local/share/conduit
 
+# Copy the Conduit binary into the image at the latest possible moment to maximise caching:
+COPY ./conduit-x86_64-unknown-linux-musl /srv/conduit/conduit
+COPY ./docker/healthcheck.sh /srv/conduit/
+
 # Add www-data user and group with UID 82, as used by alpine
 # https://git.alpinelinux.org/aports/tree/main/nginx/nginx.pre-install
 RUN set -x ; \
@@ -45,18 +55,11 @@ RUN set -x ; \
 
 # Change ownership of Conduit files to www-data user and group
 RUN chown -cR www-data:www-data /srv/conduit
+RUN chmod +x /srv/conduit/healthcheck.sh
 
-# Install packages needed to run Conduit
-RUN apk add --no-cache \
-        ca-certificates \
-        curl \
-        libgcc
 
 # Test if Conduit is still alive, uses the same endpoint as Element
-HEALTHCHECK --start-period=5s \
+HEALTHCHECK --start-period=5s --interval=60s CMD ./healthcheck.sh
-    CMD curl --fail -s "http://localhost:$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')/_matrix/client/versions" || \
-        curl -k --fail -s "https://localhost:$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')/_matrix/client/versions" || \
-        exit 1
 
 # Set user to www-data
 USER www-data
@@ -64,7 +67,3 @@ USER www-data
 WORKDIR /srv/conduit
 # Run Conduit
 ENTRYPOINT [ "/srv/conduit/conduit" ]
-
-
-# Copy the Conduit binary into the image at the latest possible moment to maximise caching:
-COPY ./conduit-x86_64-unknown-linux-musl /srv/conduit/conduit

docker/docker-compose.override.traefik.yml

@@ -10,6 +10,29 @@ services:
             - "traefik.http.routers.to-conduit.rule=Host(`<SUBDOMAIN>.<DOMAIN>`)"  # Change to the address on which Conduit is hosted
             - "traefik.http.routers.to-conduit.tls=true"
             - "traefik.http.routers.to-conduit.tls.certresolver=letsencrypt"
+            - "traefik.http.routers.to-conduit.middlewares=cors-headers@docker"
+
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOrigin=*"
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"
+
+    # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container
+    # to serve those two as static files. If you want to use a different way, delete or comment the below service, here
+    # and in the docker-compose file.
+    well-known:
+        labels:
+            - "traefik.enable=true"
+            - "traefik.docker.network=proxy"
+
+            - "traefik.http.routers.to-matrix-wellknown.rule=Host(`<SUBDOMAIN>.<DOMAIN>`) && PathPrefix(`/.well-known/matrix`)"
+            - "traefik.http.routers.to-matrix-wellknown.tls=true"
+            - "traefik.http.routers.to-matrix-wellknown.tls.certresolver=letsencrypt"
+            - "traefik.http.routers.to-matrix-wellknown.middlewares=cors-headers@docker"
+
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowOrigin=*"
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowHeaders=Origin, X-Requested-With, Content-Type, Accept, Authorization"
+            - "traefik.http.middlewares.cors-headers.headers.accessControlAllowMethods=GET, POST, PUT, DELETE, OPTIONS"
+
 
     ### Uncomment this if you uncommented Element-Web App in the docker-compose.yml
     # element-web:

docker/docker-compose.traefik.yml

@@ -27,6 +27,7 @@ services:
         environment:
             CONDUIT_SERVER_NAME: localhost:6167  # replace with your own name
             CONDUIT_TRUSTED_SERVERS: '["matrix.org"]'
+            CONDUIT_ALLOW_REGISTRATION : 'true'
             ### Uncomment and change values as desired
             # CONDUIT_ADDRESS: 0.0.0.0
             # CONDUIT_PORT: 6167
@@ -34,13 +35,22 @@ services:
             # Available levels are: error, warn, info, debug, trace - more info at: https://docs.rs/env_logger/*/env_logger/#enabling-logging
             # CONDUIT_LOG: info  # default is: "info,rocket=off,_=off,sled=off"
             # CONDUIT_ALLOW_JAEGER: 'false'
-            # CONDUIT_ALLOW_REGISTRATION : 'false'
             # CONDUIT_ALLOW_ENCRYPTION: 'false'
             # CONDUIT_ALLOW_FEDERATION: 'false'
             # CONDUIT_DATABASE_PATH: /srv/conduit/.local/share/conduit
             # CONDUIT_WORKERS: 10
             # CONDUIT_MAX_REQUEST_SIZE: 20_000_000  # in bytes, ~20 MB
 
+    # We need some way to server the client and server .well-known json. The simplest way is to use a nginx container
+    # to serve those two as static files. If you want to use a different way, delete or comment the below service, here
+    # and in the docker-compose override file.
+    well-known:
+        image: nginx:latest
+        restart: unless-stopped
+        volumes:
+            - ./nginx/matrix.conf:/etc/nginx/conf.d/matrix.conf # the config to serve the .well-known/matrix files
+            - ./nginx/www:/var/www/ # location of the client and server .well-known-files
+
     ### Uncomment if you want to use your own Element-Web App.
     ### Note: You need to provide a config.json for Element and you also need a second
     ###       Domain or Subdomain for the communication between Element and Conduit

docker/healthcheck.sh

@@ -0,0 +1,13 @@
+#!/bin/sh
+
+# If the port is not specified as env var, take it from the config file
+if [ -z ${CONDUIT_PORT} ]; then
+  CONDUIT_PORT=$(grep -m1 -o 'port\s=\s[0-9]*' conduit.toml | grep -m1 -o '[0-9]*')
+fi
+
+# The actual health check.
+# We try to first get a response on HTTP and when that fails on HTTPS and when that fails, we exit with code 1.
+# TODO: Change this to a single curl call. Do we have a config value that we can check for that?
+curl --fail -s "http://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
+    curl -k --fail -s "https://localhost:${CONDUIT_PORT}/_matrix/client/versions" || \
+    exit 1

src/appservice_server.rs

@@ -9,7 +9,7 @@ use std::{
 };
 use tracing::warn;
 
-pub async fn send_request<T: OutgoingRequest>(
+pub(crate) async fn send_request<T: OutgoingRequest>(
     globals: &crate::database::globals::Globals,
     registration: serde_yaml::Value,
     request: T,
@@ -46,7 +46,11 @@ where
     *reqwest_request.timeout_mut() = Some(Duration::from_secs(30));
 
     let url = reqwest_request.url().clone();
-    let mut response = globals.reqwest_client().execute(reqwest_request).await?;
+    let mut response = globals
+        .reqwest_client()?
+        .build()?
+        .execute(reqwest_request)
+        .await?;
 
     // reqwest::Response -> http::Response conversion
     let status = response.status();

src/client_server/account.rs

@@ -40,8 +40,12 @@ const GUEST_NAME_LENGTH: usize = 10;
 ///
 /// Checks if a username is valid and available on this server.
 ///
-/// - Returns true if no user or appservice on this server claimed this username
+/// Conditions for returning true:
-/// - This will not reserve the username, so the username might become invalid when trying to register
+/// - The user id is not historical
+/// - The server name of the user id matches this server
+/// - No user or appservice on this server already claimed this username
+///
+/// Note: This will not reserve the username, so the username might become invalid when trying to register
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/register/available", data = "<body>")
@@ -80,11 +84,15 @@ pub async fn get_register_available_route(
 ///
 /// Register an account on this homeserver.
 ///
-/// - Returns the device id and access_token unless `inhibit_login` is true
+/// You can use [`GET /_matrix/client/r0/register/available`](fn.get_register_available_route.html)
-/// - When registering a guest account, all parameters except initial_device_display_name will be
+/// to check if the user id is valid and available.
-/// ignored
+///
-/// - Creates a new account and a device for it
+/// - Only works if registration is enabled
-/// - The account will be populated with default account data
+/// - If type is guest: ignores all parameters except initial_device_display_name
+/// - If sender is not appservice: Requires UIAA (but we only use a dummy stage)
+/// - If type is not guest and no username is given: Always fails after UIAA check
+/// - Creates a new account and populates it with default account data
+/// - If `inhibit_login` is false: Creates a device and returns device id and access_token
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/register", data = "<body>")
@@ -129,7 +137,7 @@ pub async fn register_route(
     ))?;
 
     // Check if username is creative enough
-    if !missing_username && db.users.exists(&user_id)? {
+    if db.users.exists(&user_id)? {
         return Err(Error::BadRequest(
             ErrorKind::UserInUse,
             "Desired user ID is already taken.",
@@ -193,12 +201,12 @@ pub async fn register_route(
     // Create user
     db.users.create(&user_id, password)?;
 
+    // Default to pretty displayname
     let displayname = format!("{} ⚡️", user_id.localpart());
-
     db.users
         .set_displayname(&user_id, Some(displayname.clone()))?;
 
-    // Initial data
+    // Initial account data
     db.account_data.update(
         None,
         &user_id,
@@ -211,6 +219,7 @@ pub async fn register_route(
         &db.globals,
     )?;
 
+    // Inhibit login does not work for guests
     if !is_guest && body.inhibit_login {
         return Ok(register::Response {
             access_token: None,
@@ -231,7 +240,7 @@ pub async fn register_route(
     // Generate new token for the device
     let token = utils::random_string(TOKEN_LENGTH);
 
-    // Add device
+    // Create device for this account
     db.users.create_device(
         &user_id,
         &device_id,
@@ -239,7 +248,7 @@ pub async fn register_route(
         body.initial_device_display_name.clone(),
     )?;
 
-    // If this is the first user on this server, create the admins room
+    // If this is the first user on this server, create the admin room
     if db.users.count()? == 1 {
         // Create a user for the server
         let conduit_user = UserId::parse_with_server_name("conduit", db.globals.server_name())
@@ -249,6 +258,8 @@ pub async fn register_route(
 
         let room_id = RoomId::new(db.globals.server_name());
 
+        db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
+
         let mutex_state = Arc::clone(
             db.globals
                 .roomid_mutex_state
@@ -290,6 +301,7 @@ pub async fn register_route(
                     is_direct: None,
                     third_party_invite: None,
                     blurhash: None,
+                    reason: None,
                 })
                 .expect("event is valid, we just created it"),
                 unsigned: None,
@@ -455,6 +467,7 @@ pub async fn register_route(
                     is_direct: None,
                     third_party_invite: None,
                     blurhash: None,
+                    reason: None,
                 })
                 .expect("event is valid, we just created it"),
                 unsigned: None,
@@ -476,6 +489,7 @@ pub async fn register_route(
                     is_direct: None,
                     third_party_invite: None,
                     blurhash: None,
+                    reason: None,
                 })
                 .expect("event is valid, we just created it"),
                 unsigned: None,
@@ -493,8 +507,8 @@ pub async fn register_route(
             PduBuilder {
                 event_type: EventType::RoomMessage,
                 content: serde_json::to_value(message::MessageEventContent::text_html(
-                        "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing `/join #conduit:matrix.org`. **Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.** Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
+                        "## Thank you for trying out Conduit!\n\nConduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.\n\nHelpful links:\n> Website: https://conduit.rs\n> Git and Documentation: https://gitlab.com/famedly/conduit\n> Report issues: https://gitlab.com/famedly/conduit/-/issues\n\nHere are some rooms you can join (by typing the command):\n\nConduit room (Ask questions and get notified on updates):\n`/join #conduit:fachschaften.org`\n\nConduit lounge (Off-topic, only Conduit users are allowed to join)\n`/join #conduit-lounge:conduit.rs`".to_owned(),
-                        "Thanks for trying out Conduit! This software is still in development, so expect many bugs and missing features. If you have federation enabled, you can join the Conduit chat room by typing <code>/join #conduit:matrix.org</code>. <strong>Important: Please don't join any other Matrix rooms over federation without permission from the room's admins.</strong> Some actions might trigger bugs in other server implementations, breaking the chat for everyone else.".to_owned(),
+                        "<h2>Thank you for trying out Conduit!</h2>\n<p>Conduit is currently in Beta. This means you can join and participate in most Matrix rooms, but not all features are supported and you might run into bugs from time to time.</p>\n<p>Helpful links:</p>\n<blockquote>\n<p>Website: https://conduit.rs<br>Git and Documentation: https://gitlab.com/famedly/conduit<br>Report issues: https://gitlab.com/famedly/conduit/-/issues</p>\n</blockquote>\n<p>Here are some rooms you can join (by typing the command):</p>\n<p>Conduit room (Ask questions and get notified on updates):<br><code>/join #conduit:fachschaften.org</code></p>\n<p>Conduit lounge (Off-topic, only Conduit users are allowed to join)<br><code>/join #conduit-lounge:conduit.rs</code></p>\n".to_owned(),
                 ))
                 .expect("event is valid, we just created it"),
                 unsigned: None,
@@ -524,9 +538,16 @@ pub async fn register_route(
 ///
 /// Changes the password of this account.
 ///
-/// - Invalidates all other access tokens if logout_devices is true
+/// - Requires UIAA to verify user password
-/// - Deletes all other devices and most of their data (to-device events, last seen, etc.) if
+/// - Changes the password of the sender user
-/// logout_devices is true
+/// - The password hash is calculated using argon2 with 32 character salt, the plain password is
+/// not saved
+///
+/// If logout_devices is true it does the following for each device except the sender device:
+/// - Invalidates access token
+/// - Deletes device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets to-device events
+/// - Triggers device list updates
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/account/password", data = "<body>")
@@ -593,9 +614,9 @@ pub async fn change_password_route(
 
 /// # `GET _matrix/client/r0/account/whoami`
 ///
-/// Get user_id of this account.
+/// Get user_id of the sender user.
 ///
-/// - Also works for Application Services
+/// Note: Also works for Application Services
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/account/whoami", data = "<body>")
@@ -611,11 +632,13 @@ pub async fn whoami_route(body: Ruma<whoami::Request>) -> ConduitResult<whoami::
 
 /// # `POST /_matrix/client/r0/account/deactivate`
 ///
-/// Deactivate this user's account
+/// Deactivate sender user account.
 ///
 /// - Leaves all rooms and rejects all invitations
 /// - Invalidates all access tokens
-/// - Deletes all devices
+/// - Deletes all device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets all to-device events
+/// - Triggers device list updates
 /// - Removes ability to log in again
 #[cfg_attr(
     feature = "conduit_bin",
@@ -662,6 +685,7 @@ pub async fn deactivate_route(
     }
 
     // Leave all joined rooms and reject all invitations
+    // TODO: work over federation invites
     let all_rooms = db
         .rooms
         .rooms_joined(&sender_user)
@@ -681,6 +705,7 @@ pub async fn deactivate_route(
             is_direct: None,
             third_party_invite: None,
             blurhash: None,
+            reason: None,
         };
 
         let mutex_state = Arc::clone(
@@ -724,6 +749,8 @@ pub async fn deactivate_route(
 /// # `GET _matrix/client/r0/account/3pid`
 ///
 /// Get a list of third party identifiers associated with this account.
+///
+/// - Currently always returns empty list
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/account/3pid", data = "<body>")
@@ -731,7 +758,7 @@ pub async fn deactivate_route(
 pub async fn third_party_route(
     body: Ruma<get_contacts::Request>,
 ) -> ConduitResult<get_contacts::Response> {
-    let sender_user = body.sender_user.as_ref().expect("user is authenticated");
+    let _sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
     Ok(get_contacts::Response::new(Vec::new()).into())
 }

src/client_server/alias.rs

@@ -15,6 +15,9 @@ use ruma::{
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, put};
 
+/// # `PUT /_matrix/client/r0/directory/room/{roomAlias}`
+///
+/// Creates a new room alias on this server.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/directory/room/<_>", data = "<body>")
@@ -24,6 +27,13 @@ pub async fn create_alias_route(
     db: DatabaseGuard,
     body: Ruma<create_alias::Request<'_>>,
 ) -> ConduitResult<create_alias::Response> {
+    if body.room_alias.server_name() != db.globals.server_name() {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Alias is from another server.",
+        ));
+    }
+
     if db.rooms.id_from_alias(&body.room_alias)?.is_some() {
         return Err(Error::Conflict("Alias already exists."));
     }
@@ -36,6 +46,12 @@ pub async fn create_alias_route(
     Ok(create_alias::Response::new().into())
 }
 
+/// # `DELETE /_matrix/client/r0/directory/room/{roomAlias}`
+///
+/// Deletes a room alias from this server.
+///
+/// - TODO: additional access control checks
+/// - TODO: Update canonical alias event
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/r0/directory/room/<_>", data = "<body>")
@@ -45,13 +61,27 @@ pub async fn delete_alias_route(
     db: DatabaseGuard,
     body: Ruma<delete_alias::Request<'_>>,
 ) -> ConduitResult<delete_alias::Response> {
+    if body.room_alias.server_name() != db.globals.server_name() {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "Alias is from another server.",
+        ));
+    }
+
     db.rooms.set_alias(&body.room_alias, None, &db.globals)?;
 
+    // TODO: update alt_aliases?
+
     db.flush()?;
 
     Ok(delete_alias::Response::new().into())
 }
 
+/// # `GET /_matrix/client/r0/directory/room/{roomAlias}`
+///
+/// Resolve an alias locally or over federation.
+///
+/// - TODO: Suggest more servers to join via
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/directory/room/<_>", data = "<body>")
@@ -64,7 +94,7 @@ pub async fn get_alias_route(
     get_alias_helper(&db, &body.room_alias).await
 }
 
-pub async fn get_alias_helper(
+pub(crate) async fn get_alias_helper(
     db: &Database,
     room_alias: &RoomAliasId,
 ) -> ConduitResult<get_alias::Response> {

src/client_server/backup.rs

@@ -12,6 +12,9 @@ use ruma::api::client::{
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, post, put};
 
+/// # `POST /_matrix/client/r0/room_keys/version`
+///
+/// Creates a new backup.
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/unstable/room_keys/version", data = "<body>")
@@ -31,6 +34,9 @@ pub async fn create_backup_route(
     Ok(create_backup::Response { version }.into())
 }
 
+/// # `PUT /_matrix/client/r0/room_keys/version/{version}`
+///
+/// Update information about an existing backup. Only `auth_data` can be modified.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
@@ -49,6 +55,9 @@ pub async fn update_backup_route(
     Ok(update_backup::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/room_keys/version`
+///
+/// Get information about the latest backup version.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/version", data = "<body>")
@@ -77,6 +86,9 @@ pub async fn get_latest_backup_route(
     .into())
 }
 
+/// # `GET /_matrix/client/r0/room_keys/version`
+///
+/// Get information about an existing backup.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
@@ -104,6 +116,11 @@ pub async fn get_backup_route(
     .into())
 }
 
+/// # `DELETE /_matrix/client/r0/room_keys/version/{version}`
+///
+/// Delete an existing key backup.
+///
+/// - Deletes both information about the backup, as well as all key data related to the backup
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/unstable/room_keys/version/<_>", data = "<body>")
@@ -122,7 +139,13 @@ pub async fn delete_backup_route(
     Ok(delete_backup::Response {}.into())
 }
 
+/// # `PUT /_matrix/client/r0/room_keys/keys`
+///
 /// Add the received backup keys to the database.
+///
+/// - Only manipulating the most recently created version of the backup is allowed
+/// - Adds the keys to the backup
+/// - Returns the new number of keys in this backup and the etag
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/unstable/room_keys/keys", data = "<body>")
@@ -134,6 +157,18 @@ pub async fn add_backup_keys_route(
 ) -> ConduitResult<add_backup_keys::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
+    if Some(&body.version)
+        != db
+            .key_backups
+            .get_latest_backup_version(sender_user)?
+            .as_ref()
+    {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "You may only manipulate the most recently created version of the backup.",
+        ));
+    }
+
     for (room_id, room) in &body.rooms {
         for (session_id, key_data) in &room.sessions {
             db.key_backups.add_key(
@@ -156,7 +191,13 @@ pub async fn add_backup_keys_route(
     .into())
 }
 
+/// # `PUT /_matrix/client/r0/room_keys/keys/{roomId}`
+///
 /// Add the received backup keys to the database.
+///
+/// - Only manipulating the most recently created version of the backup is allowed
+/// - Adds the keys to the backup
+/// - Returns the new number of keys in this backup and the etag
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
@@ -168,6 +209,18 @@ pub async fn add_backup_key_sessions_route(
 ) -> ConduitResult<add_backup_key_sessions::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
+    if Some(&body.version)
+        != db
+            .key_backups
+            .get_latest_backup_version(sender_user)?
+            .as_ref()
+    {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "You may only manipulate the most recently created version of the backup.",
+        ));
+    }
+
     for (session_id, key_data) in &body.sessions {
         db.key_backups.add_key(
             &sender_user,
@@ -188,7 +241,13 @@ pub async fn add_backup_key_sessions_route(
     .into())
 }
 
+/// # `PUT /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}`
+///
 /// Add the received backup key to the database.
+///
+/// - Only manipulating the most recently created version of the backup is allowed
+/// - Adds the keys to the backup
+/// - Returns the new number of keys in this backup and the etag
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")
@@ -200,6 +259,18 @@ pub async fn add_backup_key_session_route(
 ) -> ConduitResult<add_backup_key_session::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
+    if Some(&body.version)
+        != db
+            .key_backups
+            .get_latest_backup_version(sender_user)?
+            .as_ref()
+    {
+        return Err(Error::BadRequest(
+            ErrorKind::InvalidParam,
+            "You may only manipulate the most recently created version of the backup.",
+        ));
+    }
+
     db.key_backups.add_key(
         &sender_user,
         &body.version,
@@ -218,6 +289,9 @@ pub async fn add_backup_key_session_route(
     .into())
 }
 
+/// # `GET /_matrix/client/r0/room_keys/keys`
+///
+/// Retrieves all keys from the backup.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/keys", data = "<body>")
@@ -234,6 +308,9 @@ pub async fn get_backup_keys_route(
     Ok(get_backup_keys::Response { rooms }.into())
 }
 
+/// # `GET /_matrix/client/r0/room_keys/keys/{roomId}`
+///
+/// Retrieves all keys from the backup for a given room.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
@@ -252,6 +329,9 @@ pub async fn get_backup_key_sessions_route(
     Ok(get_backup_key_sessions::Response { sessions }.into())
 }
 
+/// # `GET /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}`
+///
+/// Retrieves a key from the backup.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")
@@ -274,6 +354,9 @@ pub async fn get_backup_key_session_route(
     Ok(get_backup_key_session::Response { key_data }.into())
 }
 
+/// # `DELETE /_matrix/client/r0/room_keys/keys`
+///
+/// Delete the keys from the backup.
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/unstable/room_keys/keys", data = "<body>")
@@ -297,6 +380,9 @@ pub async fn delete_backup_keys_route(
     .into())
 }
 
+/// # `DELETE /_matrix/client/r0/room_keys/keys/{roomId}`
+///
+/// Delete the keys from the backup for a given room.
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/unstable/room_keys/keys/<_>", data = "<body>")
@@ -320,6 +406,9 @@ pub async fn delete_backup_key_sessions_route(
     .into())
 }
 
+/// # `DELETE /_matrix/client/r0/room_keys/keys/{roomId}/{sessionId}`
+///
+/// Delete a key from the backup.
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/unstable/room_keys/keys/<_>/<_>", data = "<body>")

src/client_server/capabilities.rs

@@ -13,7 +13,7 @@ use rocket::get;
 
 /// # `GET /_matrix/client/r0/capabilities`
 ///
-/// Get information on this server's supported feature set and other relevent capabilities.
+/// Get information on the supported feature set and other relevent capabilities of this server.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/capabilities", data = "<_body>")

src/client_server/config.rs

@@ -16,6 +16,9 @@ use serde_json::{json, value::RawValue as RawJsonValue};
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
 
+/// # `PUT /_matrix/client/r0/user/{userId}/account_data/{type}`
+///
+/// Sets some account data for the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/user/<_>/account_data/<_>", data = "<body>")
@@ -48,6 +51,9 @@ pub async fn set_global_account_data_route(
     Ok(set_global_account_data::Response {}.into())
 }
 
+/// # `PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/account_data/{type}`
+///
+/// Sets some room account data for the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     put(
@@ -83,6 +89,9 @@ pub async fn set_room_account_data_route(
     Ok(set_room_account_data::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/user/{userId}/account_data/{type}`
+///
+/// Gets some account data for the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/user/<_>/account_data/<_>", data = "<body>")
@@ -98,7 +107,6 @@ pub async fn get_global_account_data_route(
         .account_data
         .get::<Box<RawJsonValue>>(None, sender_user, body.event_type.clone().into())?
         .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
-    db.flush()?;
 
     let account_data = serde_json::from_str::<ExtractGlobalEventContent>(event.get())
         .map_err(|_| Error::bad_database("Invalid account data event in db."))?
@@ -107,6 +115,9 @@ pub async fn get_global_account_data_route(
     Ok(get_global_account_data::Response { account_data }.into())
 }
 
+/// # `GET /_matrix/client/r0/user/{userId}/rooms/{roomId}/account_data/{type}`
+///
+/// Gets some room account data for the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     get(
@@ -129,7 +140,6 @@ pub async fn get_room_account_data_route(
             body.event_type.clone().into(),
         )?
         .ok_or(Error::BadRequest(ErrorKind::NotFound, "Data not found."))?;
-    db.flush()?;
 
     let account_data = serde_json::from_str::<ExtractRoomEventContent>(event.get())
         .map_err(|_| Error::bad_database("Invalid account data event in db."))?

src/client_server/context.rs

@@ -5,6 +5,12 @@ use std::convert::TryFrom;
 #[cfg(feature = "conduit_bin")]
 use rocket::get;
 
+/// # `GET /_matrix/client/r0/rooms/{roomId}/context`
+///
+/// Allows loading room history around an event.
+///
+/// - Only works if the user is joined (TODO: always allow, but only show events if the user was
+/// joined, depending on history_visibility)
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/context/<_>", data = "<body>")
@@ -44,7 +50,7 @@ pub async fn get_context_route(
 
     let events_before = db
         .rooms
-        .pdus_until(&sender_user, &body.room_id, base_token)
+        .pdus_until(&sender_user, &body.room_id, base_token)?
         .take(
             u32::try_from(body.limit).map_err(|_| {
                 Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.")
@@ -66,7 +72,7 @@ pub async fn get_context_route(
 
     let events_after = db
         .rooms
-        .pdus_after(&sender_user, &body.room_id, base_token)
+        .pdus_after(&sender_user, &body.room_id, base_token)?
         .take(
             u32::try_from(body.limit).map_err(|_| {
                 Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.")

src/client_server/device.rs

@@ -11,6 +11,9 @@ use super::SESSION_ID_LENGTH;
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, post, put};
 
+/// # `GET /_matrix/client/r0/devices`
+///
+/// Get metadata on all devices of the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/devices", data = "<body>")
@@ -31,6 +34,9 @@ pub async fn get_devices_route(
     Ok(get_devices::Response { devices }.into())
 }
 
+/// # `GET /_matrix/client/r0/devices/{deviceId}`
+///
+/// Get metadata on a single device of the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/devices/<_>", data = "<body>")
@@ -50,6 +56,9 @@ pub async fn get_device_route(
     Ok(get_device::Response { device }.into())
 }
 
+/// # `PUT /_matrix/client/r0/devices/{deviceId}`
+///
+/// Updates the metadata on a given device of the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/devices/<_>", data = "<body>")
@@ -76,6 +85,15 @@ pub async fn update_device_route(
     Ok(update_device::Response {}.into())
 }
 
+/// # `PUT /_matrix/client/r0/devices/{deviceId}`
+///
+/// Deletes the given device.
+///
+/// - Requires UIAA to verify user password
+/// - Invalidates access token
+/// - Deletes device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets to-device events
+/// - Triggers device list updates
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/r0/devices/<_>", data = "<body>")
@@ -128,6 +146,17 @@ pub async fn delete_device_route(
     Ok(delete_device::Response {}.into())
 }
 
+/// # `PUT /_matrix/client/r0/devices/{deviceId}`
+///
+/// Deletes the given device.
+///
+/// - Requires UIAA to verify user password
+///
+/// For each device:
+/// - Invalidates access token
+/// - Deletes device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets to-device events
+/// - Triggers device list updates
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/delete_devices", data = "<body>")

src/client_server/directory.rs

@@ -28,6 +28,11 @@ use tracing::{info, warn};
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post, put};
 
+/// # `POST /_matrix/client/r0/publicRooms`
+///
+/// Lists the public rooms on this server.
+///
+/// - Rooms are ordered by the number of joined members
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/publicRooms", data = "<body>")
@@ -48,6 +53,11 @@ pub async fn get_public_rooms_filtered_route(
     .await
 }
 
+/// # `GET /_matrix/client/r0/publicRooms`
+///
+/// Lists the public rooms on this server.
+///
+/// - Rooms are ordered by the number of joined members
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/publicRooms", data = "<body>")
@@ -77,6 +87,11 @@ pub async fn get_public_rooms_route(
     .into())
 }
 
+/// # `PUT /_matrix/client/r0/directory/list/room/{roomId}`
+///
+/// Sets the visibility of a given room in the room directory.
+///
+/// - TODO: Access control checks
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/directory/list/room/<_>", data = "<body>")
@@ -107,6 +122,9 @@ pub async fn set_room_visibility_route(
     Ok(set_room_visibility::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/directory/list/room/{roomId}`
+///
+/// Gets the visibility of a given room in the room directory.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/directory/list/room/<_>", data = "<body>")
@@ -126,7 +144,7 @@ pub async fn get_room_visibility_route(
     .into())
 }
 
-pub async fn get_public_rooms_filtered_helper(
+pub(crate) async fn get_public_rooms_filtered_helper(
     db: &Database,
     server: Option<&ServerName>,
     limit: Option<UInt>,
@@ -233,8 +251,7 @@ pub async fn get_public_rooms_filtered_helper(
                             .map_err(|_| {
                                 Error::bad_database("Invalid room name event in database.")
                             })?
-                            .name
+                            .name)
-                            .map(|n| n.to_owned().into()))
                         })?,
                     num_joined_members: db
                         .rooms

src/client_server/filter.rs

@@ -4,6 +4,9 @@ use ruma::api::client::r0::filter::{self, create_filter, get_filter};
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};
 
+/// # `GET /_matrix/client/r0/user/{userId}/filter/{filterId}`
+///
+/// TODO: Loads a filter that was previously created.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/user/<_>/filter/<_>"))]
 #[tracing::instrument]
 pub async fn get_filter_route() -> ConduitResult<get_filter::Response> {
@@ -18,6 +21,9 @@ pub async fn get_filter_route() -> ConduitResult<get_filter::Response> {
     .into())
 }
 
+/// # `PUT /_matrix/client/r0/user/{userId}/filter`
+///
+/// TODO: Creates a new filter to be used by other endpoints.
 #[cfg_attr(feature = "conduit_bin", post("/_matrix/client/r0/user/<_>/filter"))]
 #[tracing::instrument]
 pub async fn create_filter_route() -> ConduitResult<create_filter::Response> {

src/client_server/keys.rs

@@ -1,5 +1,6 @@
 use super::SESSION_ID_LENGTH;
 use crate::{database::DatabaseGuard, utils, ConduitResult, Database, Error, Result, Ruma};
+use rocket::futures::{prelude::*, stream::FuturesUnordered};
 use ruma::{
     api::{
         client::{
@@ -18,11 +19,17 @@ use ruma::{
     DeviceId, DeviceKeyAlgorithm, UserId,
 };
 use serde_json::json;
-use std::collections::{BTreeMap, HashSet};
+use std::collections::{BTreeMap, HashMap, HashSet};
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};
 
+/// # `POST /_matrix/client/r0/keys/upload`
+///
+/// Publish end-to-end encryption keys for the sender device.
+///
+/// - Adds one time keys
+/// - If there are no device keys yet: Adds device keys (TODO: merge with existing keys?)
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/keys/upload", data = "<body>")
@@ -48,6 +55,7 @@ pub async fn upload_keys_route(
     }
 
     if let Some(device_keys) = &body.device_keys {
+        // TODO: merge this and the existing event?
         // This check is needed to assure that signatures are kept
         if db
             .users
@@ -72,6 +80,13 @@ pub async fn upload_keys_route(
     .into())
 }
 
+/// # `POST /_matrix/client/r0/keys/query`
+///
+/// Get end-to-end encryption keys for the given users.
+///
+/// - Always fetches users from other servers over federation
+/// - Gets master keys, self-signing keys, user signing keys and device keys.
+/// - The master and self-signing keys contain signatures that the user is allowed to see
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/keys/query", data = "<body>")
@@ -94,6 +109,9 @@ pub async fn get_keys_route(
     Ok(response.into())
 }
 
+/// # `POST /_matrix/client/r0/keys/claim`
+///
+/// Claims one-time keys
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/keys/claim", data = "<body>")
@@ -110,6 +128,11 @@ pub async fn claim_keys_route(
     Ok(response.into())
 }
 
+/// # `POST /_matrix/client/r0/keys/device_signing/upload`
+///
+/// Uploads end-to-end key information for the sender user.
+///
+/// - Requires UIAA to verify password
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/unstable/keys/device_signing/upload", data = "<body>")
@@ -171,6 +194,9 @@ pub async fn upload_signing_keys_route(
     Ok(upload_signing_keys::Response {}.into())
 }
 
+/// # `POST /_matrix/client/r0/keys/signatures/upload`
+///
+/// Uploads end-to-end key signatures from the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/unstable/keys/signatures/upload", data = "<body>")
@@ -232,6 +258,11 @@ pub async fn upload_signatures_route(
     Ok(upload_signatures::Response {}.into())
 }
 
+/// # `POST /_matrix/client/r0/keys/changes`
+///
+/// Gets a list of users who have updated their device identity keys since the previous sync token.
+///
+/// - TODO: left users
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/keys/changes", data = "<body>")
@@ -283,7 +314,7 @@ pub async fn get_key_changes_route(
     .into())
 }
 
-pub async fn get_keys_helper<F: Fn(&UserId) -> bool>(
+pub(crate) async fn get_keys_helper<F: Fn(&UserId) -> bool>(
     sender_user: Option<&UserId>,
     device_keys_input: &BTreeMap<UserId, Vec<Box<DeviceId>>>,
     allowed_signatures: F,
@@ -294,7 +325,7 @@ pub async fn get_keys_helper<F: Fn(&UserId) -> bool>(
     let mut user_signing_keys = BTreeMap::new();
     let mut device_keys = BTreeMap::new();
 
-    let mut get_over_federation = BTreeMap::new();
+    let mut get_over_federation = HashMap::new();
 
     for (user_id, device_ids) in device_keys_input {
         if user_id.server_name() != db.globals.server_name() {
@@ -364,13 +395,16 @@ pub async fn get_keys_helper<F: Fn(&UserId) -> bool>(
 
     let mut failures = BTreeMap::new();
 
-    for (server, vec) in get_over_federation {
+    let mut futures = get_over_federation
+        .into_iter()
+        .map(|(server, vec)| async move {
             let mut device_keys_input_fed = BTreeMap::new();
             for (user_id, keys) in vec {
                 device_keys_input_fed.insert(user_id.clone(), keys.clone());
             }
-        match db
+            (
-            .sending
+                server,
+                db.sending
                     .send_federation_request(
                         &db.globals,
                         server,
@@ -378,8 +412,13 @@ pub async fn get_keys_helper<F: Fn(&UserId) -> bool>(
                             device_keys: device_keys_input_fed,
                         },
                     )
-            .await
+                    .await,
-        {
+            )
+        })
+        .collect::<FuturesUnordered<_>>();
+
+    while let Some((server, response)) = futures.next().await {
+        match response {
             Ok(response) => {
                 master_keys.extend(response.master_keys);
                 self_signing_keys.extend(response.self_signing_keys);
@@ -400,7 +439,7 @@ pub async fn get_keys_helper<F: Fn(&UserId) -> bool>(
     })
 }
 
-pub async fn claim_keys_helper(
+pub(crate) async fn claim_keys_helper(
     one_time_keys_input: &BTreeMap<UserId, BTreeMap<Box<DeviceId>, DeviceKeyAlgorithm>>,
     db: &Database,
 ) -> Result<claim_keys::Response> {
@@ -430,13 +469,15 @@ pub async fn claim_keys_helper(
         one_time_keys.insert(user_id.clone(), container);
     }
 
+    let mut failures = BTreeMap::new();
+
     for (server, vec) in get_over_federation {
         let mut one_time_keys_input_fed = BTreeMap::new();
         for (user_id, keys) in vec {
             one_time_keys_input_fed.insert(user_id.clone(), keys.clone());
         }
         // Ignore failures
-        let keys = db
+        if let Ok(keys) = db
             .sending
             .send_federation_request(
                 &db.globals,
@@ -445,13 +486,16 @@ pub async fn claim_keys_helper(
                     one_time_keys: one_time_keys_input_fed,
                 },
             )
-            .await?;
+            .await
-
+        {
             one_time_keys.extend(keys.one_time_keys);
+        } else {
+            failures.insert(server.to_string(), json!({}));
+        }
     }
 
     Ok(claim_keys::Response {
-        failures: BTreeMap::new(),
+        failures,
         one_time_keys,
     })
 }

src/client_server/media.rs

@@ -12,6 +12,9 @@ use rocket::{get, post};
 
 const MXC_LENGTH: usize = 32;
 
+/// # `GET /_matrix/media/r0/config`
+///
+/// Returns max upload size.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/media/r0/config"))]
 #[tracing::instrument(skip(db))]
 pub async fn get_media_config_route(
@@ -23,6 +26,12 @@ pub async fn get_media_config_route(
     .into())
 }
 
+/// # `POST /_matrix/media/r0/upload`
+///
+/// Permanently save media in the server.
+///
+/// - Some metadata will be saved in the database
+/// - Media will be saved in the media/ directory
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/media/r0/upload", data = "<body>")
@@ -61,6 +70,11 @@ pub async fn create_content_route(
     .into())
 }
 
+/// # `POST /_matrix/media/r0/download/{serverName}/{mediaId}`
+///
+/// Load media from our server or over federation.
+///
+/// - Only allows federation if `allow_remote` is true
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/media/r0/download/<_>/<_>", data = "<body>")
@@ -114,6 +128,11 @@ pub async fn get_content_route(
     }
 }
 
+/// # `POST /_matrix/media/r0/thumbnail/{serverName}/{mediaId}`
+///
+/// Load media thumbnail from our server or over federation.
+///
+/// - Only allows federation if `allow_remote` is true
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/media/r0/thumbnail/<_>/<_>", data = "<body>")

src/client_server/membership.rs

@@ -5,7 +5,6 @@ use crate::{
     server_server, utils, ConduitResult, Database, Error, Result, Ruma,
 };
 use member::{MemberEventContent, MembershipState};
-use rocket::futures;
 use ruma::{
     api::{
         client::{
@@ -38,6 +37,12 @@ use tracing::{debug, error, warn};
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/join`
+///
+/// Tries to join the sender user into a room.
+///
+/// - If the server knowns about this room: creates the join event and does auth rules locally
+/// - If the server does not know about the room: asks other servers over federation
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/join", data = "<body>")
@@ -79,6 +84,12 @@ pub async fn join_room_by_id_route(
     ret
 }
 
+/// # `POST /_matrix/client/r0/join/{roomIdOrAlias}`
+///
+/// Tries to join the sender user into a room.
+///
+/// - If the server knowns about this room: creates the join event and does auth rules locally
+/// - If the server does not know about the room: asks other servers over federation
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/join/<_>", data = "<body>")
@@ -133,6 +144,11 @@ pub async fn join_room_by_id_or_alias_route(
     .into())
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/leave`
+///
+/// Tries to leave the sender user from a room.
+///
+/// - This should always work if the user is currently joined.
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/leave", data = "<body>")
@@ -151,6 +167,9 @@ pub async fn leave_room_route(
     Ok(leave_room::Response::new().into())
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/invite`
+///
+/// Tries to send an invite event into the room.
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/invite", data = "<body>")
@@ -171,6 +190,9 @@ pub async fn invite_user_route(
     }
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/kick`
+///
+/// Tries to send a kick event into the room.
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/kick", data = "<body>")
@@ -234,6 +256,9 @@ pub async fn kick_user_route(
     Ok(kick_user::Response::new().into())
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/ban`
+///
+/// Tries to send a ban event into the room.
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/ban", data = "<body>")
@@ -262,6 +287,7 @@ pub async fn ban_user_route(
                 is_direct: None,
                 third_party_invite: None,
                 blurhash: db.users.blurhash(&body.user_id)?,
+                reason: None,
             }),
             |event| {
                 let mut event = serde_json::from_value::<Raw<member::MemberEventContent>>(
@@ -306,6 +332,9 @@ pub async fn ban_user_route(
     Ok(ban_user::Response::new().into())
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/unban`
+///
+/// Tries to send an unban event into the room.
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/unban", data = "<body>")
@@ -368,6 +397,14 @@ pub async fn unban_user_route(
     Ok(unban_user::Response::new().into())
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/forget`
+///
+/// Forgets about a room.
+///
+/// - If the sender user currently left the room: Stops sender user from receiving information about the room
+///
+/// Note: Other devices of the user have no way of knowing the room was forgotten, so this has to
+/// be called from every device
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/forget", data = "<body>")
@@ -386,6 +423,9 @@ pub async fn forget_room_route(
     Ok(forget_room::Response::new().into())
 }
 
+/// # `POST /_matrix/client/r0/joined_rooms`
+///
+/// Lists all rooms the user has joined.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/joined_rooms", data = "<body>")
@@ -407,6 +447,11 @@ pub async fn joined_rooms_route(
     .into())
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/members`
+///
+/// Lists all joined users in a room (TODO: at a specific point in time, with a specific membership).
+///
+/// - Only works if the user is currently joined
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/members", data = "<body>")
@@ -418,6 +463,7 @@ pub async fn get_member_events_route(
 ) -> ConduitResult<get_member_events::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
+    // TODO: check history visibility?
     if !db.rooms.is_joined(sender_user, &body.room_id)? {
         return Err(Error::BadRequest(
             ErrorKind::Forbidden,
@@ -437,6 +483,12 @@ pub async fn get_member_events_route(
     .into())
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/joined_members`
+///
+/// Lists all members of a room.
+///
+/// - The sender user must be in the room
+/// - TODO: An appservice just needs a puppet joined
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/joined_members", data = "<body>")
@@ -448,11 +500,7 @@ pub async fn joined_members_route(
 ) -> ConduitResult<joined_members::Response> {
     let sender_user = body.sender_user.as_ref().expect("user is authenticated");
 
-    if !db
+    if !db.rooms.is_joined(&sender_user, &body.room_id)? {
-        .rooms
-        .is_joined(&sender_user, &body.room_id)
-        .unwrap_or(false)
-    {
         return Err(Error::BadRequest(
             ErrorKind::Forbidden,
             "You aren't a member of the room.",
@@ -563,6 +611,7 @@ async fn join_room_by_id_helper(
                 is_direct: None,
                 third_party_invite: None,
                 blurhash: db.users.blurhash(&sender_user)?,
+                reason: None,
             })
             .expect("event is valid, we just created it"),
         );
@@ -609,20 +658,27 @@ async fn join_room_by_id_helper(
             )
             .await?;
 
+        db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
+
         let pdu = PduEvent::from_id_val(&event_id, join_event.clone())
             .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?;
 
         let mut state = HashMap::new();
         let pub_key_map = RwLock::new(BTreeMap::new());
 
-        for result in futures::future::join_all(
+        server_server::fetch_join_signing_keys(
-            send_join_response
+            &send_join_response,
+            &room_version,
+            &pub_key_map,
+            &db,
+        )
+        .await?;
+
+        for result in send_join_response
             .room_state
             .state
             .iter()
-                .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db)),
+            .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db))
-        )
-        .await
         {
             let (event_id, value) = match result {
                 Ok(t) => t,
@@ -636,32 +692,46 @@ async fn join_room_by_id_helper(
 
             db.rooms.add_pdu_outlier(&event_id, &value)?;
             if let Some(state_key) = &pdu.state_key {
-                state.insert((pdu.kind.clone(), state_key.clone()), pdu.event_id.clone());
+                let shortstatekey =
+                    db.rooms
+                        .get_or_create_shortstatekey(&pdu.kind, state_key, &db.globals)?;
+                state.insert(shortstatekey, pdu.event_id.clone());
             }
         }
 
-        state.insert(
+        let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey(
-            (
+            &pdu.kind,
-                pdu.kind.clone(),
+            pdu.state_key
-                pdu.state_key.clone().expect("join event has state key"),
+                .as_ref()
-            ),
+                .expect("Pdu is a membership state event"),
-            pdu.event_id.clone(),
+            &db.globals,
-        );
+        )?;
 
-        if state.get(&(EventType::RoomCreate, "".to_owned())).is_none() {
+        state.insert(incoming_shortstatekey, pdu.event_id.clone());
+
+        let create_shortstatekey = db
+            .rooms
+            .get_shortstatekey(&EventType::RoomCreate, "")?
+            .expect("Room exists");
+
+        if state.get(&create_shortstatekey).is_none() {
             return Err(Error::BadServerResponse("State contained no create event."));
         }
 
-        db.rooms.force_state(room_id, state, &db)?;
+        db.rooms.force_state(
+            room_id,
+            state
+                .into_iter()
+                .map(|(k, id)| db.rooms.compress_state_event(k, &id, &db.globals))
+                .collect::<Result<HashSet<_>>>()?,
+            &db,
+        )?;
 
-        for result in futures::future::join_all(
+        for result in send_join_response
-            send_join_response
             .room_state
             .auth_chain
             .iter()
-                .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db)),
+            .map(|pdu| validate_and_add_event_id(pdu, &room_version, &pub_key_map, &db))
-        )
-        .await
         {
             let (event_id, value) = match result {
                 Ok(t) => t,
@@ -693,6 +763,7 @@ async fn join_room_by_id_helper(
             is_direct: None,
             third_party_invite: None,
             blurhash: db.users.blurhash(&sender_user)?,
+            reason: None,
         };
 
         db.rooms.build_and_append_pdu(
@@ -717,7 +788,7 @@ async fn join_room_by_id_helper(
     Ok(join_room_by_id::Response::new(room_id.clone()).into())
 }
 
-async fn validate_and_add_event_id(
+fn validate_and_add_event_id(
     pdu: &Raw<Pdu>,
     room_version: &RoomVersionId,
     pub_key_map: &RwLock<BTreeMap<String, BTreeMap<String, String>>>,
@@ -760,7 +831,6 @@ async fn validate_and_add_event_id(
         }
     }
 
-    server_server::fetch_required_signing_keys(&value, pub_key_map, db).await?;
     if let Err(e) = ruma::signatures::verify_event(
         &*pub_key_map
             .read()
@@ -781,7 +851,7 @@ async fn validate_and_add_event_id(
     Ok((event_id, value))
 }
 
-pub async fn invite_helper<'a>(
+pub(crate) async fn invite_helper<'a>(
     sender_user: &UserId,
     user_id: &UserId,
     room_id: &RoomId,
@@ -817,7 +887,10 @@ pub async fn invite_helper<'a>(
                     serde_json::from_value::<Raw<CreateEventContent>>(create_event.content.clone())
                         .expect("Raw::from_value always works.")
                         .deserialize()
-                        .map_err(|_| Error::bad_database("Invalid PowerLevels event in db."))
+                        .map_err(|e| {
+                            warn!("Invalid create event: {}", e);
+                            Error::bad_database("Invalid create event in db.")
+                        })
                 })
                 .transpose()?;
 
@@ -844,6 +917,7 @@ pub async fn invite_helper<'a>(
                 membership: MembershipState::Invite,
                 third_party_invite: None,
                 blurhash: None,
+                reason: None,
             })
             .expect("member event is valid value");
 
@@ -904,8 +978,8 @@ pub async fn invite_helper<'a>(
                 &room_version,
                 &Arc::new(pdu.clone()),
                 create_prev_event,
-                &auth_events,
                 None, // TODO: third_party_invite
+                |k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone),
             )
             .map_err(|e| {
                 error!("{:?}", e);
@@ -947,6 +1021,14 @@ pub async fn invite_helper<'a>(
             (room_version_id, pdu_json, invite_room_state)
         };
 
+        // Generate event id
+        let expected_event_id = EventId::try_from(&*format!(
+            "${}",
+            ruma::signatures::reference_hash(&pdu_json, &room_version_id)
+                .expect("ruma can calculate reference hashes")
+        ))
+        .expect("ruma's reference hashes are valid event ids");
+
         let response = db
             .sending
             .send_federation_request(
@@ -954,9 +1036,9 @@ pub async fn invite_helper<'a>(
                 user_id.server_name(),
                 create_invite::v2::Request {
                     room_id: room_id.clone(),
-                    event_id: ruma::event_id!("$receivingservershouldsetthis"),
+                    event_id: expected_event_id.clone(),
                     room_version: room_version_id,
-                    event: PduEvent::convert_to_outgoing_federation_event(pdu_json),
+                    event: PduEvent::convert_to_outgoing_federation_event(pdu_json.clone()),
                     invite_room_state,
                 },
             )
@@ -976,6 +1058,10 @@ pub async fn invite_helper<'a>(
             }
         };
 
+        if expected_event_id != event_id {
+            warn!("Server {} changed invite event, that's not allowed in the spec: ours: {:?}, theirs: {:?}", user_id.server_name(), pdu_json, value);
+        }
+
         let origin = serde_json::from_value::<Box<ServerName>>(
             serde_json::to_value(value.get("origin").ok_or(Error::BadRequest(
                 ErrorKind::InvalidParam,
@@ -1038,6 +1124,7 @@ pub async fn invite_helper<'a>(
                 is_direct: Some(is_direct),
                 third_party_invite: None,
                 blurhash: db.users.blurhash(&user_id)?,
+                reason: None,
             })
             .expect("event is valid, we just created it"),
             unsigned: None,

src/client_server/message.rs

@@ -16,6 +16,13 @@ use std::{
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
 
+/// # `PUT /_matrix/client/r0/rooms/{roomId}/send/{eventType}/{txnId}`
+///
+/// Send a message event into the room.
+///
+/// - Is a NOOP if the txn id was already used before and returns the same event id again
+/// - The only requirement for the content is that it has to be valid json
+/// - Tries to send the event into the room, auth rules will determine if it is allowed
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/send/<_>/<_>", data = "<body>")
@@ -92,6 +99,12 @@ pub async fn send_message_event_route(
     Ok(send_message_event::Response::new(event_id).into())
 }
 
+/// # `GET /_matrix/client/r0/rooms/{roomId}/messages`
+///
+/// Allows paginating through room history.
+///
+/// - Only works if the user is joined (TODO: always allow, but only show events where the user was
+/// joined, depending on history_visibility)
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/messages", data = "<body>")
@@ -128,7 +141,7 @@ pub async fn get_message_events_route(
         get_message_events::Direction::Forward => {
             let events_after = db
                 .rooms
-                .pdus_after(&sender_user, &body.room_id, from)
+                .pdus_after(&sender_user, &body.room_id, from)?
                 .take(limit)
                 .filter_map(|r| r.ok()) // Filter out buggy events
                 .filter_map(|(pdu_id, pdu)| {
@@ -158,7 +171,7 @@ pub async fn get_message_events_route(
         get_message_events::Direction::Backward => {
             let events_before = db
                 .rooms
-                .pdus_until(&sender_user, &body.room_id, from)
+                .pdus_until(&sender_user, &body.room_id, from)?
                 .take(limit)
                 .filter_map(|r| r.ok()) // Filter out buggy events
                 .filter_map(|(pdu_id, pdu)| {

src/client_server/mod.rs

@@ -71,6 +71,9 @@ pub const DEVICE_ID_LENGTH: usize = 10;
 pub const TOKEN_LENGTH: usize = 256;
 pub const SESSION_ID_LENGTH: usize = 256;
 
+/// # `OPTIONS`
+///
+/// Web clients use this to get CORS headers.
 #[cfg(feature = "conduit_bin")]
 #[options("/<_..>")]
 #[tracing::instrument]

src/client_server/presence.rs

@@ -5,6 +5,9 @@ use std::{convert::TryInto, time::Duration};
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
 
+/// # `PUT /_matrix/client/r0/presence/{userId}/status`
+///
+/// Sets the presence state of the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/presence/<_>/status", data = "<body>")
@@ -46,6 +49,11 @@ pub async fn set_presence_route(
     Ok(set_presence::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/presence/{userId}/status`
+///
+/// Gets the presence state of the given user.
+///
+/// - Only works if you share a room with the user
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/presence/<_>/status", data = "<body>")
@@ -71,6 +79,7 @@ pub async fn get_presence_route(
             .get_last_presence_event(&sender_user, &room_id)?
         {
             presence_event = Some(presence);
+            break;
         }
     }
 

src/client_server/profile.rs

@@ -17,6 +17,11 @@ use std::{convert::TryInto, sync::Arc};
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
 
+/// # `PUT /_matrix/client/r0/profile/{userId}/displayname`
+///
+/// Updates the displayname.
+///
+/// - Also makes sure other users receive the update using presence EDUs
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/profile/<_>/displayname", data = "<body>")
@@ -32,10 +37,9 @@ pub async fn set_displayname_route(
         .set_displayname(&sender_user, body.displayname.clone())?;
 
     // Send a new membership event and presence update into all joined rooms
-    let all_rooms_joined = db.rooms.rooms_joined(&sender_user).collect::<Vec<_>>();
+    let all_rooms_joined: Vec<_> = db
-
+        .rooms
-    for (pdu_builder, room_id) in all_rooms_joined
+        .rooms_joined(&sender_user)
-        .into_iter()
         .filter_map(|r| r.ok())
         .map(|room_id| {
             Ok::<_, Error>((
@@ -72,7 +76,9 @@ pub async fn set_displayname_route(
             ))
         })
         .filter_map(|r| r.ok())
-    {
+        .collect();
+
+    for (pdu_builder, room_id) in all_rooms_joined {
         let mutex_state = Arc::clone(
             db.globals
                 .roomid_mutex_state
@@ -115,6 +121,11 @@ pub async fn set_displayname_route(
     Ok(set_display_name::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/profile/{userId}/displayname`
+///
+/// Returns the displayname of the user.
+///
+/// - If user is on another server: Fetches displayname over federation
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/profile/<_>/displayname", data = "<body>")
@@ -149,6 +160,11 @@ pub async fn get_displayname_route(
     .into())
 }
 
+/// # `PUT /_matrix/client/r0/profile/{userId}/avatar_url`
+///
+/// Updates the avatar_url and blurhash.
+///
+/// - Also makes sure other users receive the update using presence EDUs
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/profile/<_>/avatar_url", data = "<body>")
@@ -166,10 +182,9 @@ pub async fn set_avatar_url_route(
     db.users.set_blurhash(&sender_user, body.blurhash.clone())?;
 
     // Send a new membership event and presence update into all joined rooms
-    let all_joined_rooms = db.rooms.rooms_joined(&sender_user).collect::<Vec<_>>();
+    let all_joined_rooms: Vec<_> = db
-
+        .rooms
-    for (pdu_builder, room_id) in all_joined_rooms
+        .rooms_joined(&sender_user)
-        .into_iter()
         .filter_map(|r| r.ok())
         .map(|room_id| {
             Ok::<_, Error>((
@@ -206,7 +221,9 @@ pub async fn set_avatar_url_route(
             ))
         })
         .filter_map(|r| r.ok())
-    {
+        .collect();
+
+    for (pdu_builder, room_id) in all_joined_rooms {
         let mutex_state = Arc::clone(
             db.globals
                 .roomid_mutex_state
@@ -249,6 +266,11 @@ pub async fn set_avatar_url_route(
     Ok(set_avatar_url::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/profile/{userId}/avatar_url`
+///
+/// Returns the avatar_url and blurhash of the user.
+///
+/// - If user is on another server: Fetches avatar_url and blurhash over federation
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/profile/<_>/avatar_url", data = "<body>")
@@ -285,6 +307,11 @@ pub async fn get_avatar_url_route(
     .into())
 }
 
+/// # `GET /_matrix/client/r0/profile/{userId}`
+///
+/// Returns the displayname, avatar_url and blurhash of the user.
+///
+/// - If user is on another server: Fetches profile over federation
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/profile/<_>", data = "<body>")

src/client_server/push.rs

@@ -15,6 +15,9 @@ use ruma::{
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, post, put};
 
+/// # `GET /_matrix/client/r0/pushrules`
+///
+/// Retrieves the push rules event for this user.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/pushrules", data = "<body>")
@@ -40,6 +43,9 @@ pub async fn get_pushrules_all_route(
     .into())
 }
 
+/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
+///
+/// Retrieves a single specified push rule for this user.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
@@ -94,6 +100,9 @@ pub async fn get_pushrule_route(
     }
 }
 
+/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
+///
+/// Creates a single specified push rule for this user.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<req>")
@@ -197,6 +206,9 @@ pub async fn set_pushrule_route(
     Ok(set_pushrule::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/actions`
+///
+/// Gets the actions of a single specified push rule for this user.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "<body>")
@@ -256,6 +268,9 @@ pub async fn get_pushrule_actions_route(
     .into())
 }
 
+/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/actions`
+///
+/// Sets the actions of a single specified push rule for this user.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/actions", data = "<body>")
@@ -330,6 +345,9 @@ pub async fn set_pushrule_actions_route(
     Ok(set_pushrule_actions::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/enabled`
+///
+/// Gets the enabled status of a single specified push rule for this user.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "<body>")
@@ -391,6 +409,9 @@ pub async fn get_pushrule_enabled_route(
     Ok(get_pushrule_enabled::Response { enabled }.into())
 }
 
+/// # `PUT /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}/enabled`
+///
+/// Sets the enabled status of a single specified push rule for this user.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/pushrules/<_>/<_>/<_>/enabled", data = "<body>")
@@ -470,6 +491,9 @@ pub async fn set_pushrule_enabled_route(
     Ok(set_pushrule_enabled::Response {}.into())
 }
 
+/// # `DELETE /_matrix/client/r0/pushrules/{scope}/{kind}/{ruleId}`
+///
+/// Deletes a single specified push rule for this user.
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/r0/pushrules/<_>/<_>/<_>", data = "<body>")
@@ -539,6 +563,9 @@ pub async fn delete_pushrule_route(
     Ok(delete_pushrule::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/pushers`
+///
+/// Gets all currently active pushers for the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/pushers", data = "<body>")
@@ -556,6 +583,11 @@ pub async fn get_pushers_route(
     .into())
 }
 
+/// # `POST /_matrix/client/r0/pushers/set`
+///
+/// Adds a pusher for the sender user.
+///
+/// - TODO: Handle `append`
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/pushers/set", data = "<body>")

src/client_server/read_marker.rs

@@ -13,6 +13,12 @@ use std::collections::BTreeMap;
 #[cfg(feature = "conduit_bin")]
 use rocket::post;
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/read_markers`
+///
+/// Sets different types of read markers.
+///
+/// - Updates fully-read account data event to `fully_read`
+/// - If `read_receipt` is set: Update private marker and public read receipt EDU
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/read_markers", data = "<body>")
@@ -80,6 +86,9 @@ pub async fn set_read_marker_route(
     Ok(set_read_marker::Response {}.into())
 }
 
+/// # `POST /_matrix/client/r0/rooms/{roomId}/receipt/{receiptType}/{eventId}`
+///
+/// Sets private read marker and public read receipt EDU.
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/receipt/<_>/<_>", data = "<body>")

src/client_server/redact.rs

@@ -9,6 +9,11 @@ use ruma::{
 #[cfg(feature = "conduit_bin")]
 use rocket::put;
 
+/// # `PUT /_matrix/client/r0/rooms/{roomId}/redact/{eventId}/{txnId}`
+///
+/// Tries to send a redaction event into the room.
+///
+/// - TODO: Handle txn id
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/redact/<_>/<_>", data = "<body>")

src/client_server/room.rs

@@ -20,6 +20,22 @@ use tracing::{info, warn};
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, post};
 
+/// # `POST /_matrix/client/r0/createRoom`
+///
+/// Creates a new room.
+///
+/// - Room ID is randomly generated
+/// - Create alias if room_alias_name is set
+/// - Send create event
+/// - Join sender user
+/// - Send power levels event
+/// - Send canonical room alias
+/// - Send join rules
+/// - Send history visibility
+/// - Send guest access
+/// - Send events listed in initial state
+/// - Send events implied by `name` and `topic`
+/// - Send invite events
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/createRoom", data = "<body>")
@@ -33,6 +49,8 @@ pub async fn create_room_route(
 
     let room_id = RoomId::new(db.globals.server_name());
 
+    db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
+
     let mutex_state = Arc::clone(
         db.globals
             .roomid_mutex_state
@@ -43,8 +61,8 @@ pub async fn create_room_route(
     );
     let state_lock = mutex_state.lock().await;
 
-    let alias = body
+    let alias: Option<RoomAliasId> =
-        .room_alias_name
+        body.room_alias_name
             .as_ref()
             .map_or(Ok(None), |localpart| {
                 // TODO: Check for invalid characters and maximum length
@@ -105,6 +123,7 @@ pub async fn create_room_route(
                 is_direct: Some(body.is_direct),
                 third_party_invite: None,
                 blurhash: db.users.blurhash(&sender_user)?,
+                reason: None,
             })
             .expect("event is valid, we just created it"),
             unsigned: None,
@@ -172,9 +191,32 @@ pub async fn create_room_route(
         &state_lock,
     )?;
 
-    // 4. Events set by preset
+    // 4. Canonical room alias
+    if let Some(room_alias_id) = &alias {
+        db.rooms.build_and_append_pdu(
+            PduBuilder {
+                event_type: EventType::RoomCanonicalAlias,
+                content: serde_json::to_value(
+                    ruma::events::room::canonical_alias::CanonicalAliasEventContent {
+                        alias: Some(room_alias_id.clone()),
+                        alt_aliases: vec![],
+                    },
+                )
+                .expect("We checked that alias earlier, it must be fine"),
+                unsigned: None,
+                state_key: Some("".to_owned()),
+                redacts: None,
+            },
+            &sender_user,
+            &room_id,
+            &db,
+            &state_lock,
+        )?;
+    }
 
-    // 4.1 Join Rules
+    // 5. Events set by preset
+
+    // 5.1 Join Rules
     db.rooms.build_and_append_pdu(
         PduBuilder {
             event_type: EventType::RoomJoinRules,
@@ -199,7 +241,7 @@ pub async fn create_room_route(
         &state_lock,
     )?;
 
-    // 4.2 History Visibility
+    // 5.2 History Visibility
     db.rooms.build_and_append_pdu(
         PduBuilder {
             event_type: EventType::RoomHistoryVisibility,
@@ -217,7 +259,7 @@ pub async fn create_room_route(
         &state_lock,
     )?;
 
-    // 4.3 Guest Access
+    // 5.3 Guest Access
     db.rooms.build_and_append_pdu(
         PduBuilder {
             event_type: EventType::RoomGuestAccess,
@@ -243,7 +285,7 @@ pub async fn create_room_route(
         &state_lock,
     )?;
 
-    // 5. Events listed in initial_state
+    // 6. Events listed in initial_state
     for event in &body.initial_state {
         let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| {
             warn!("Invalid initial state event: {:?}", e);
@@ -259,7 +301,7 @@ pub async fn create_room_route(
             .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock)?;
     }
 
-    // 6. Events implied by name and topic
+    // 7. Events implied by name and topic
     if let Some(name) = &body.name {
         db.rooms.build_and_append_pdu(
             PduBuilder {
@@ -296,7 +338,7 @@ pub async fn create_room_route(
         )?;
     }
 
-    // 7. Events implied by invite (and TODO: invite_3pid)
+    // 8. Events implied by invite (and TODO: invite_3pid)
     drop(state_lock);
     for user_id in &body.invite {
         let _ = invite_helper(sender_user, user_id, &room_id, &db, body.is_direct).await;
@@ -318,6 +360,11 @@ pub async fn create_room_route(
     Ok(create_room::Response::new(room_id).into())
 }
 
+/// # `GET /_matrix/client/r0/rooms/{roomId}/event/{eventId}`
+///
+/// Gets a single event.
+///
+/// - You have to currently be joined to the room (TODO: Respect history visibility)
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/event/<_>", data = "<body>")
@@ -346,6 +393,11 @@ pub async fn get_room_event_route(
     .into())
 }
 
+/// # `GET /_matrix/client/r0/rooms/{roomId}/aliases`
+///
+/// Lists all aliases of the room.
+///
+/// - Only users joined to the room are allowed to call this TODO: Allow any user to call it if history_visibility is world readable
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/aliases", data = "<body>")
@@ -374,6 +426,16 @@ pub async fn get_room_aliases_route(
     .into())
 }
 
+/// # `GET /_matrix/client/r0/rooms/{roomId}/upgrade`
+///
+/// Upgrades the room.
+///
+/// - Creates a replacement room
+/// - Sends a tombstone event into the current room
+/// - Sender user joins the room
+/// - Transfers some state events
+/// - Moves local aliases
+/// - Modifies old room power levels to prevent users from speaking
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/rooms/<_>/upgrade", data = "<body>")
@@ -397,6 +459,8 @@ pub async fn upgrade_room_route(
 
     // Create a replacement room
     let replacement_room = RoomId::new(db.globals.server_name());
+    db.rooms
+        .get_or_create_shortroomid(&replacement_room, &db.globals)?;
 
     let mutex_state = Arc::clone(
         db.globals
@@ -492,6 +556,7 @@ pub async fn upgrade_room_route(
                 is_direct: None,
                 third_party_invite: None,
                 blurhash: db.users.blurhash(&sender_user)?,
+                reason: None,
             })
             .expect("event is valid, we just created it"),
             unsigned: None,

src/client_server/search.rs

@@ -6,6 +6,11 @@ use rocket::post;
 use search_events::{EventContextResult, ResultCategories, ResultRoomEvents, SearchResult};
 use std::collections::BTreeMap;
 
+/// # `POST /_matrix/client/r0/search`
+///
+/// Searches rooms for messages.
+///
+/// - Only works if the user is currently joined to the room (TODO: Respect history visibility)
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/search", data = "<body>")

src/client_server/session.rs

@@ -3,7 +3,10 @@ use crate::{database::DatabaseGuard, utils, ConduitResult, Error, Ruma};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::session::{get_login_types, login, logout, logout_all},
+        r0::{
+            session::{get_login_types, login, logout, logout_all},
+            uiaa::IncomingUserIdentifier,
+        },
     },
     UserId,
 };
@@ -21,7 +24,7 @@ use rocket::{get, post};
 
 /// # `GET /_matrix/client/r0/login`
 ///
-/// Get the homeserver's supported login types. One of these should be used as the `type` field
+/// Get the supported login types of this server. One of these should be used as the `type` field
 /// when logging in.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/login"))]
 #[tracing::instrument]
@@ -38,9 +41,10 @@ pub async fn get_login_types_route() -> ConduitResult<get_login_types::Response>
 ///
 /// Authenticates the user and returns an access token it can use in subsequent requests.
 ///
-/// - The returned access token is associated with the user and device
+/// - The user needs to authenticate using their password (or if enabled using a json web token)
-/// - Old access tokens of that device should be invalidated
+/// - If `device_id` is known: invalidates old access token of that device
-/// - If `device_id` is unknown, a new device will be created
+/// - If `device_id` is unknown: creates a new device
+/// - Returns access token that is associated with the user and device
 ///
 /// Note: You can use [`GET /_matrix/client/r0/login`](fn.get_supported_versions_route.html) to see
 /// supported login types.
@@ -60,7 +64,7 @@ pub async fn login_route(
             identifier,
             password,
         } => {
-            let username = if let login::IncomingUserIdentifier::MatrixId(matrix_id) = identifier {
+            let username = if let IncomingUserIdentifier::MatrixId(matrix_id) = identifier {
                 matrix_id
             } else {
                 return Err(Error::BadRequest(ErrorKind::Forbidden, "Bad login type."));
@@ -159,8 +163,10 @@ pub async fn login_route(
 ///
 /// Log out the current device.
 ///
-/// - Invalidates the access token
+/// - Invalidates access token
-/// - Deletes the device and most of it's data (to-device events, last seen, etc.)
+/// - Deletes device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets to-device events
+/// - Triggers device list updates
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/logout", data = "<body>")
@@ -185,7 +191,9 @@ pub async fn logout_route(
 /// Log out all devices of this user.
 ///
 /// - Invalidates all access tokens
-/// - Deletes devices and most of their data (to-device events, last seen, etc.)
+/// - Deletes all device metadata (device id, device display name, last seen ip, last seen ts)
+/// - Forgets all to-device events
+/// - Triggers device list updates
 ///
 /// Note: This is equivalent to calling [`GET /_matrix/client/r0/logout`](fn.logout_route.html)
 /// from each device of this user.

src/client_server/state.rs

@@ -22,6 +22,13 @@ use ruma::{
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, put};
 
+/// # `PUT /_matrix/client/r0/rooms/{roomId}/state/{eventType}/{stateKey}`
+///
+/// Sends a state event into the room.
+///
+/// - The only requirement for the content is that it has to be valid json
+/// - Tries to send the event into the room, auth rules will determine if it is allowed
+/// - If event is new canonical_alias: Rejects if alias is incorrect
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "<body>")
@@ -48,6 +55,13 @@ pub async fn send_state_event_for_key_route(
     Ok(send_state_event::Response { event_id }.into())
 }
 
+/// # `PUT /_matrix/client/r0/rooms/{roomId}/state/{eventType}`
+///
+/// Sends a state event into the room.
+///
+/// - The only requirement for the content is that it has to be valid json
+/// - Tries to send the event into the room, auth rules will determine if it is allowed
+/// - If event is new canonical_alias: Rejects if alias is incorrect
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/state/<_>", data = "<body>")
@@ -74,6 +88,11 @@ pub async fn send_state_event_for_empty_key_route(
     Ok(send_state_event::Response { event_id }.into())
 }
 
+/// # `GET /_matrix/client/r0/rooms/{roomid}/state`
+///
+/// Get all state events for a room.
+///
+/// - If not joined: Only works if current room history visibility is world readable
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/state", data = "<body>")
@@ -121,6 +140,11 @@ pub async fn get_state_events_route(
     .into())
 }
 
+/// # `GET /_matrix/client/r0/rooms/{roomid}/state/{eventType}/{stateKey}`
+///
+/// Get single state event of a room.
+///
+/// - If not joined: Only works if current room history visibility is world readable
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/state/<_>/<_>", data = "<body>")
@@ -172,6 +196,11 @@ pub async fn get_state_events_for_key_route(
     .into())
 }
 
+/// # `GET /_matrix/client/r0/rooms/{roomid}/state/{eventType}`
+///
+/// Get single state event of a room.
+///
+/// - If not joined: Only works if current room history visibility is world readable
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/rooms/<_>/state/<_>", data = "<body>")
@@ -223,7 +252,7 @@ pub async fn get_state_events_for_empty_key_route(
     .into())
 }
 
-pub async fn send_state_event_for_key_helper(
+async fn send_state_event_for_key_helper(
     db: &Database,
     sender: &UserId,
     room_id: &RoomId,
@@ -233,6 +262,8 @@ pub async fn send_state_event_for_key_helper(
 ) -> Result<EventId> {
     let sender_user = sender;
 
+    // TODO: Review this check, error if event is unparsable, use event type, allow alias if it
+    // previously existed
     if let Ok(canonical_alias) =
         serde_json::from_str::<CanonicalAliasEventContent>(json.json().get())
     {

src/client_server/sync.rs

@@ -12,7 +12,7 @@ use std::{
     time::Duration,
 };
 use tokio::sync::watch::Sender;
-use tracing::{error, warn};
+use tracing::error;
 
 #[cfg(feature = "conduit_bin")]
 use rocket::{get, tokio};
@@ -22,12 +22,33 @@ use rocket::{get, tokio};
 /// Synchronize the client's state with the latest state on the server.
 ///
 /// - This endpoint takes a `since` parameter which should be the `next_batch` value from a
-/// previous request.
+/// previous request for incremental syncs.
-/// - Calling this endpoint without a `since` parameter will return all recent events, the state
+///
-/// of all rooms and more data. This should only be called on the initial login of the device.
+/// Calling this endpoint without a `since` parameter returns:
-/// - To get incremental updates, you can call this endpoint with a `since` parameter. This will
+/// - Some of the most recent events of each timeline
-/// return all recent events, state updates and more data that happened since the last /sync
+/// - Notification counts for each room
-/// request.
+/// - Joined and invited member counts, heroes
+/// - All state events
+///
+/// Calling this endpoint with a `since` parameter from a previous `next_batch` returns:
+/// For joined rooms:
+/// - Some of the most recent events of each timeline that happened after since
+/// - If user joined the room after since: All state events and device list updates in that room
+/// - If the user was already in the room: A list of all events that are in the state now, but were
+/// not in the state at `since`
+/// - If the state we send contains a member event: Joined and invited member counts, heroes
+/// - Device list updates that happened after `since`
+/// - If there are events in the timeline we send or the user send updated his read mark: Notification counts
+/// - EDUs that are active now (read receipts, typing updates, presence)
+///
+/// For invited rooms:
+/// - If the user was invited after `since`: A subset of the state of the room at the point of the invite
+///
+/// For left rooms:
+/// - If the user left after `since`: prev_batch token, empty state (TODO: subset of the state at the point of the leave)
+///
+/// - Sync is handled in an async task, multiple requests from the same device with the same
+/// `since` will be cached
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/sync", data = "<body>")
@@ -106,7 +127,7 @@ pub async fn sync_events_route(
     result
 }
 
-pub async fn sync_helper_wrapper(
+async fn sync_helper_wrapper(
     db: Arc<DatabaseGuard>,
     sender_user: UserId,
     sender_device: Box<DeviceId>,
@@ -205,7 +226,7 @@ async fn sync_helper(
 
         let mut non_timeline_pdus = db
             .rooms
-            .pdus_until(&sender_user, &room_id, u64::MAX)
+            .pdus_until(&sender_user, &room_id, u64::MAX)?
             .filter_map(|r| {
                 // Filter out buggy events
                 if r.is_err() {
@@ -246,36 +267,12 @@ async fn sync_helper(
             .current_shortstatehash(&room_id)?
             .expect("All rooms have state");
 
-        let first_pdu_before_since = db
+        let since_shortstatehash = db.rooms.get_token_shortstatehash(&room_id, since)?;
-            .rooms
-            .pdus_until(&sender_user, &room_id, since)
-            .next()
-            .transpose()?;
-
-        let pdus_after_since = db
-            .rooms
-            .pdus_after(&sender_user, &room_id, since)
-            .next()
-            .is_some();
-
-        let since_shortstatehash = first_pdu_before_since
-            .as_ref()
-            .map(|pdu| {
-                db.rooms
-                    .pdu_shortstatehash(&pdu.1.event_id)
-                    .transpose()
-                    .ok_or_else(|| {
-                        warn!("PDU without state: {}", pdu.1.event_id);
-                        Error::bad_database("Found PDU without state")
-                    })
-            })
-            .transpose()?
-            .transpose()?;
 
         // Calculates joined_member_count, invited_member_count and heroes
         let calculate_counts = || {
-            let joined_member_count = db.rooms.room_members(&room_id).count();
+            let joined_member_count = db.rooms.room_joined_count(&room_id)?.unwrap_or(0);
-            let invited_member_count = db.rooms.room_members_invited(&room_id).count();
+            let invited_member_count = db.rooms.room_invited_count(&room_id)?.unwrap_or(0);
 
             // Recalculate heroes (first 5 members)
             let mut heroes = Vec::new();
@@ -286,7 +283,7 @@ async fn sync_helper(
 
                 for hero in db
                     .rooms
-                    .all_pdus(&sender_user, &room_id)
+                    .all_pdus(&sender_user, &room_id)?
                     .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus
                     .filter(|(_, pdu)| pdu.kind == EventType::RoomMember)
                     .map(|(_, pdu)| {
@@ -328,11 +325,11 @@ async fn sync_helper(
                 }
             }
 
-            (
+            Ok::<_, Error>((
                 Some(joined_member_count),
                 Some(invited_member_count),
                 heroes,
-            )
+            ))
         };
 
         let (
@@ -343,12 +340,12 @@ async fn sync_helper(
             state_events,
         ) = if since_shortstatehash.is_none() {
             // Probably since = 0, we will do an initial sync
-            let (joined_member_count, invited_member_count, heroes) = calculate_counts();
+            let (joined_member_count, invited_member_count, heroes) = calculate_counts()?;
 
             let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?;
             let state_events = current_state_ids
                 .iter()
-                .map(|id| db.rooms.get_pdu(id))
+                .map(|(_, id)| db.rooms.get_pdu(id))
                 .filter_map(|r| r.ok().flatten())
                 .collect::<Vec<_>>();
 
@@ -359,7 +356,7 @@ async fn sync_helper(
                 true,
                 state_events,
             )
-        } else if !pdus_after_since || since_shortstatehash == Some(current_shortstatehash) {
+        } else if timeline_pdus.is_empty() && since_shortstatehash == Some(current_shortstatehash) {
             // No state changes
             (Vec::new(), None, None, false, Vec::new())
         } else {
@@ -393,18 +390,14 @@ async fn sync_helper(
             let state_events = if joined_since_last_sync {
                 current_state_ids
                     .iter()
-                    .map(|id| db.rooms.get_pdu(id))
+                    .map(|(_, id)| db.rooms.get_pdu(id))
                     .filter_map(|r| r.ok().flatten())
                     .collect::<Vec<_>>()
             } else {
                 current_state_ids
-                    .difference(&since_state_ids)
-                    .filter(|id| {
-                        !timeline_pdus
                     .iter()
-                            .any(|(_, timeline_pdu)| timeline_pdu.event_id == **id)
+                    .filter(|(key, id)| since_state_ids.get(key) != Some(id))
-                    })
+                    .map(|(_, id)| db.rooms.get_pdu(id))
-                    .map(|id| db.rooms.get_pdu(id))
                     .filter_map(|r| r.ok().flatten())
                     .collect()
             };
@@ -423,66 +416,38 @@ async fn sync_helper(
 
             let send_member_count = state_events
                 .iter()
-                .any(|event| event.kind == EventType::RoomMember)
+                .any(|event| event.kind == EventType::RoomMember);
-                || timeline_pdus.iter().any(|(_, event)| {
-                    event.state_key.is_some() && event.kind == EventType::RoomMember
-                });
 
             if encrypted_room {
-                for (user_id, current_member) in db
+                for state_event in &state_events {
-                    .rooms
+                    if state_event.kind != EventType::RoomMember {
-                    .room_members(&room_id)
+                        continue;
-                    .filter_map(|r| r.ok())
+                    }
-                    .filter_map(|user_id| {
+
-                        db.rooms
+                    if let Some(state_key) = &state_event.state_key {
-                            .state_get(
+                        let user_id = UserId::try_from(state_key.clone())
-                                current_shortstatehash,
+                            .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?;
-                                &EventType::RoomMember,
+
-                                user_id.as_str(),
+                        if user_id == sender_user {
-                            )
+                            continue;
-                            .ok()
+                        }
-                            .flatten()
+
-                            .map(|current_member| (user_id, current_member))
+                        let new_membership = serde_json::from_value::<
-                    })
-                {
-                    let current_membership = serde_json::from_value::<
                             Raw<ruma::events::room::member::MemberEventContent>,
-                    >(current_member.content.clone())
+                        >(state_event.content.clone())
                         .expect("Raw::from_value always works")
                         .deserialize()
                         .map_err(|_| Error::bad_database("Invalid PDU in database."))?
                         .membership;
 
-                    let since_membership = db
+                        match new_membership {
-                        .rooms
+                            MembershipState::Join => {
-                        .state_get(
-                            since_shortstatehash,
-                            &EventType::RoomMember,
-                            user_id.as_str(),
-                        )?
-                        .and_then(|since_member| {
-                            serde_json::from_value::<
-                                Raw<ruma::events::room::member::MemberEventContent>,
-                            >(since_member.content.clone())
-                            .expect("Raw::from_value always works")
-                            .deserialize()
-                            .map_err(|_| Error::bad_database("Invalid PDU in database."))
-                            .ok()
-                        })
-                        .map_or(MembershipState::Leave, |member| member.membership);
-
-                    let user_id = UserId::try_from(user_id.clone())
-                        .map_err(|_| Error::bad_database("Invalid UserId in member PDU."))?;
-
-                    match (since_membership, current_membership) {
-                        (MembershipState::Leave, MembershipState::Join) => {
                                 // A new user joined an encrypted room
                                 if !share_encrypted_room(&db, &sender_user, &user_id, &room_id)? {
                                     device_list_updates.insert(user_id);
                                 }
                             }
-                        // TODO: Remove, this should never happen here, right?
+                            MembershipState::Leave => {
-                        (MembershipState::Join, MembershipState::Leave) => {
                                 // Write down users that have left encrypted rooms we are in
                                 left_encrypted_users.insert(user_id);
                             }
@@ -490,6 +455,7 @@ async fn sync_helper(
                         }
                     }
                 }
+            }
 
             if joined_since_last_sync && encrypted_room || new_encrypted_room {
                 // If the user is in a new encrypted room, give them all joined users
@@ -510,7 +476,7 @@ async fn sync_helper(
             }
 
             let (joined_member_count, invited_member_count, heroes) = if send_member_count {
-                calculate_counts()
+                calculate_counts()?
             } else {
                 (None, None, Vec::new())
             };
@@ -584,6 +550,10 @@ async fn sync_helper(
             );
         }
 
+        // Save the state after this sync so we can send the correct state diff next sync
+        db.rooms
+            .associate_token_shortstatehash(&room_id, next_batch, current_shortstatehash)?;
+
         let joined_room = sync_events::JoinedRoom {
             account_data: sync_events::RoomAccountData {
                 events: db

src/client_server/tag.rs

@@ -8,6 +8,11 @@ use std::collections::BTreeMap;
 #[cfg(feature = "conduit_bin")]
 use rocket::{delete, get, put};
 
+/// # `PUT /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags/{tag}`
+///
+/// Adds a tag to the room.
+///
+/// - Inserts the tag into the tag event of the room account data.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/user/<_>/rooms/<_>/tags/<_>", data = "<body>")
@@ -45,6 +50,11 @@ pub async fn update_tag_route(
     Ok(create_tag::Response {}.into())
 }
 
+/// # `DELETE /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags/{tag}`
+///
+/// Deletes a tag from the room.
+///
+/// - Removes the tag from the tag event of the room account data.
 #[cfg_attr(
     feature = "conduit_bin",
     delete("/_matrix/client/r0/user/<_>/rooms/<_>/tags/<_>", data = "<body>")
@@ -79,6 +89,11 @@ pub async fn delete_tag_route(
     Ok(delete_tag::Response {}.into())
 }
 
+/// # `GET /_matrix/client/r0/user/{userId}/rooms/{roomId}/tags`
+///
+/// Returns tags on the room.
+///
+/// - Gets the tag event of the room account data.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/user/<_>/rooms/<_>/tags", data = "<body>")

src/client_server/thirdparty.rs

@@ -5,6 +5,9 @@ use ruma::api::client::r0::thirdparty::get_protocols;
 use rocket::get;
 use std::collections::BTreeMap;
 
+/// # `GET /_matrix/client/r0/thirdparty/protocols`
+///
+/// TODO: Fetches all metadata about protocols supported by the homeserver.
 #[cfg_attr(
     feature = "conduit_bin",
     get("/_matrix/client/r0/thirdparty/protocols")

src/client_server/to_device.rs

@@ -13,6 +13,9 @@ use ruma::{
 #[cfg(feature = "conduit_bin")]
 use rocket::put;
 
+/// # `PUT /_matrix/client/r0/sendToDevice/{eventType}/{txnId}`
+///
+/// Send a to-device event to a set of client devices.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/sendToDevice/<_>/<_>", data = "<body>")
@@ -56,6 +59,7 @@ pub async fn send_event_to_device_route(
                         },
                     ))
                     .expect("DirectToDevice EDU can be serialized"),
+                    db.globals.next_count()?,
                 )?;
 
                 continue;

src/client_server/typing.rs

@@ -5,6 +5,9 @@ use ruma::api::client::r0::typing::create_typing_event;
 #[cfg(feature = "conduit_bin")]
 use rocket::put;
 
+/// # `PUT /_matrix/client/r0/rooms/{roomId}/typing/{userId}`
+///
+/// Sets the typing state of the sender user.
 #[cfg_attr(
     feature = "conduit_bin",
     put("/_matrix/client/r0/rooms/<_>/typing/<_>", data = "<body>")

src/client_server/unversioned.rs

@@ -10,7 +10,7 @@ use rocket::get;
 ///
 /// - Versions take the form MAJOR.MINOR.PATCH
 /// - Only the latest PATCH release will be reported for each MAJOR.MINOR value
-/// - Unstable features should be namespaced and may include version information in their name
+/// - Unstable features are namespaced and may include version information in their name
 ///
 /// Note: Unstable features are used while developing new features. Clients should avoid using
 /// unstable features in their stable releases

src/client_server/user_directory.rs

@@ -4,6 +4,11 @@ use ruma::api::client::r0::user_directory::search_users;
 #[cfg(feature = "conduit_bin")]
 use rocket::post;
 
+/// # `POST /_matrix/client/r0/user_directory/search`
+///
+/// Searches all known users for a match.
+///
+/// - TODO: Hide users that are not in any public rooms?
 #[cfg_attr(
     feature = "conduit_bin",
     post("/_matrix/client/r0/user_directory/search", data = "<body>")

src/client_server/voip.rs

@@ -5,6 +5,9 @@ use std::time::Duration;
 #[cfg(feature = "conduit_bin")]
 use rocket::get;
 
+/// # `GET /_matrix/client/r0/voip/turnServer`
+///
+/// TODO: Returns information about the recommended turn server.
 #[cfg_attr(feature = "conduit_bin", get("/_matrix/client/r0/voip/turnServer"))]
 #[tracing::instrument]
 pub async fn turn_server_route() -> ConduitResult<get_turn_server_info::Response> {

src/database.rs

@@ -24,13 +24,14 @@ use rocket::{
     request::{FromRequest, Request},
     Shutdown, State,
 };
-use ruma::{DeviceId, RoomId, ServerName, UserId};
+use ruma::{DeviceId, EventId, RoomId, ServerName, UserId};
 use serde::{de::IgnoredAny, Deserialize};
 use std::{
-    collections::{BTreeMap, HashMap},
+    collections::{BTreeMap, HashMap, HashSet},
-    convert::TryFrom,
+    convert::{TryFrom, TryInto},
     fs::{self, remove_dir_all},
     io::Write,
+    mem::size_of,
     ops::Deref,
     path::Path,
     sync::{Arc, Mutex, RwLock},
@@ -46,13 +47,15 @@ pub struct Config {
     database_path: String,
     #[serde(default = "default_db_cache_capacity_mb")]
     db_cache_capacity_mb: f64,
+    #[serde(default = "default_pdu_cache_capacity")]
+    pdu_cache_capacity: u32,
     #[serde(default = "default_sqlite_wal_clean_second_interval")]
     sqlite_wal_clean_second_interval: u32,
     #[serde(default = "default_max_request_size")]
     max_request_size: u32,
     #[serde(default = "default_max_concurrent_requests")]
     max_concurrent_requests: u16,
-    #[serde(default = "true_fn")]
+    #[serde(default = "false_fn")]
     allow_registration: bool,
     #[serde(default = "true_fn")]
     allow_encryption: bool,
@@ -106,8 +109,12 @@ fn default_db_cache_capacity_mb() -> f64 {
     200.0
 }
 
+fn default_pdu_cache_capacity() -> u32 {
+    100_000
+}
+
 fn default_sqlite_wal_clean_second_interval() -> u32 {
-    15 * 60 // every 15 minutes
+    1 * 60 // every minute
 }
 
 fn default_max_request_size() -> u32 {
@@ -251,6 +258,7 @@ impl Database {
                 userroomid_joined: builder.open_tree("userroomid_joined")?,
                 roomuserid_joined: builder.open_tree("roomuserid_joined")?,
                 roomid_joinedcount: builder.open_tree("roomid_joinedcount")?,
+                roomid_invitedcount: builder.open_tree("roomid_invitedcount")?,
                 roomuseroncejoinedids: builder.open_tree("roomuseroncejoinedids")?,
                 userroomid_invitestate: builder.open_tree("userroomid_invitestate")?,
                 roomuserid_invitecount: builder.open_tree("roomuserid_invitecount")?,
@@ -261,17 +269,38 @@ impl Database {
                 userroomid_highlightcount: builder.open_tree("userroomid_highlightcount")?,
 
                 statekey_shortstatekey: builder.open_tree("statekey_shortstatekey")?,
-                stateid_shorteventid: builder.open_tree("stateid_shorteventid")?,
+                shortstatekey_statekey: builder.open_tree("shortstatekey_statekey")?,
+
+                shorteventid_authchain: builder.open_tree("shorteventid_authchain")?,
+
+                roomid_shortroomid: builder.open_tree("roomid_shortroomid")?,
+
+                shortstatehash_statediff: builder.open_tree("shortstatehash_statediff")?,
                 eventid_shorteventid: builder.open_tree("eventid_shorteventid")?,
                 shorteventid_eventid: builder.open_tree("shorteventid_eventid")?,
                 shorteventid_shortstatehash: builder.open_tree("shorteventid_shortstatehash")?,
                 roomid_shortstatehash: builder.open_tree("roomid_shortstatehash")?,
+                roomsynctoken_shortstatehash: builder.open_tree("roomsynctoken_shortstatehash")?,
                 statehash_shortstatehash: builder.open_tree("statehash_shortstatehash")?,
 
                 eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?,
+                softfailedeventids: builder.open_tree("softfailedeventids")?,
+
                 referencedevents: builder.open_tree("referencedevents")?,
-                pdu_cache: Mutex::new(LruCache::new(1_000_000)),
+                pdu_cache: Mutex::new(LruCache::new(
+                    config
+                        .pdu_cache_capacity
+                        .try_into()
+                        .expect("pdu cache capacity fits into usize"),
+                )),
                 auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
+                shorteventid_cache: Mutex::new(LruCache::new(1_000_000)),
+                eventidshort_cache: Mutex::new(LruCache::new(1_000_000)),
+                shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)),
+                statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)),
+                our_real_users_cache: RwLock::new(HashMap::new()),
+                appservice_in_room_cache: RwLock::new(HashMap::new()),
+                stateinfo_cache: Mutex::new(LruCache::new(1000)),
             },
             account_data: account_data::AccountData {
                 roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,
@@ -424,19 +453,289 @@ impl Database {
             }
 
             if db.globals.database_version()? < 6 {
-                // TODO update to 6
                 // Set room member count
                 for (roomid, _) in db.rooms.roomid_shortstatehash.iter() {
                     let room_id =
                         RoomId::try_from(utils::string_from_bytes(&roomid).unwrap()).unwrap();
 
-                    db.rooms.update_joined_count(&room_id)?;
+                    db.rooms.update_joined_count(&room_id, &db)?;
                 }
 
                 db.globals.bump_database_version(6)?;
 
                 println!("Migration: 5 -> 6 finished");
             }
+
+            if db.globals.database_version()? < 7 {
+                // Upgrade state store
+                let mut last_roomstates: HashMap<RoomId, u64> = HashMap::new();
+                let mut current_sstatehash: Option<u64> = None;
+                let mut current_room = None;
+                let mut current_state = HashSet::new();
+                let mut counter = 0;
+
+                let mut handle_state =
+                    |current_sstatehash: u64,
+                     current_room: &RoomId,
+                     current_state: HashSet<_>,
+                     last_roomstates: &mut HashMap<_, _>| {
+                        counter += 1;
+                        println!("counter: {}", counter);
+                        let last_roomsstatehash = last_roomstates.get(current_room);
+
+                        let states_parents = last_roomsstatehash.map_or_else(
+                            || Ok(Vec::new()),
+                            |&last_roomsstatehash| {
+                                db.rooms.load_shortstatehash_info(dbg!(last_roomsstatehash))
+                            },
+                        )?;
+
+                        let (statediffnew, statediffremoved) =
+                            if let Some(parent_stateinfo) = states_parents.last() {
+                                let statediffnew = current_state
+                                    .difference(&parent_stateinfo.1)
+                                    .cloned()
+                                    .collect::<HashSet<_>>();
+
+                                let statediffremoved = parent_stateinfo
+                                    .1
+                                    .difference(&current_state)
+                                    .cloned()
+                                    .collect::<HashSet<_>>();
+
+                                (statediffnew, statediffremoved)
+                            } else {
+                                (current_state, HashSet::new())
+                            };
+
+                        db.rooms.save_state_from_diff(
+                            dbg!(current_sstatehash),
+                            statediffnew,
+                            statediffremoved,
+                            2, // every state change is 2 event changes on average
+                            states_parents,
+                        )?;
+
+                        /*
+                        let mut tmp = db.rooms.load_shortstatehash_info(&current_sstatehash, &db)?;
+                        let state = tmp.pop().unwrap();
+                        println!(
+                            "{}\t{}{:?}: {:?} + {:?} - {:?}",
+                            current_room,
+                            "  ".repeat(tmp.len()),
+                            utils::u64_from_bytes(&current_sstatehash).unwrap(),
+                            tmp.last().map(|b| utils::u64_from_bytes(&b.0).unwrap()),
+                            state
+                                .2
+                                .iter()
+                                .map(|b| utils::u64_from_bytes(&b[size_of::<u64>()..]).unwrap())
+                                .collect::<Vec<_>>(),
+                            state
+                                .3
+                                .iter()
+                                .map(|b| utils::u64_from_bytes(&b[size_of::<u64>()..]).unwrap())
+                                .collect::<Vec<_>>()
+                        );
+                        */
+
+                        Ok::<_, Error>(())
+                    };
+
+                for (k, seventid) in db._db.open_tree("stateid_shorteventid")?.iter() {
+                    let sstatehash = utils::u64_from_bytes(&k[0..size_of::<u64>()])
+                        .expect("number of bytes is correct");
+                    let sstatekey = k[size_of::<u64>()..].to_vec();
+                    if Some(sstatehash) != current_sstatehash {
+                        if let Some(current_sstatehash) = current_sstatehash {
+                            handle_state(
+                                current_sstatehash,
+                                current_room.as_ref().unwrap(),
+                                current_state,
+                                &mut last_roomstates,
+                            )?;
+                            last_roomstates
+                                .insert(current_room.clone().unwrap(), current_sstatehash);
+                        }
+                        current_state = HashSet::new();
+                        current_sstatehash = Some(sstatehash);
+
+                        let event_id = db
+                            .rooms
+                            .shorteventid_eventid
+                            .get(&seventid)
+                            .unwrap()
+                            .unwrap();
+                        let event_id =
+                            EventId::try_from(utils::string_from_bytes(&event_id).unwrap())
+                                .unwrap();
+                        let pdu = db.rooms.get_pdu(&event_id).unwrap().unwrap();
+
+                        if Some(&pdu.room_id) != current_room.as_ref() {
+                            current_room = Some(pdu.room_id.clone());
+                        }
+                    }
+
+                    let mut val = sstatekey;
+                    val.extend_from_slice(&seventid);
+                    current_state.insert(val.try_into().expect("size is correct"));
+                }
+
+                if let Some(current_sstatehash) = current_sstatehash {
+                    handle_state(
+                        current_sstatehash,
+                        current_room.as_ref().unwrap(),
+                        current_state,
+                        &mut last_roomstates,
+                    )?;
+                }
+
+                db.globals.bump_database_version(7)?;
+
+                println!("Migration: 6 -> 7 finished");
+            }
+
+            if db.globals.database_version()? < 8 {
+                // Generate short room ids for all rooms
+                for (room_id, _) in db.rooms.roomid_shortstatehash.iter() {
+                    let shortroomid = db.globals.next_count()?.to_be_bytes();
+                    db.rooms.roomid_shortroomid.insert(&room_id, &shortroomid)?;
+                    println!("Migration: 8");
+                }
+                // Update pduids db layout
+                let mut batch = db.rooms.pduid_pdu.iter().filter_map(|(key, v)| {
+                    if !key.starts_with(b"!") {
+                        return None;
+                    }
+                    let mut parts = key.splitn(2, |&b| b == 0xff);
+                    let room_id = parts.next().unwrap();
+                    let count = parts.next().unwrap();
+
+                    let short_room_id = db
+                        .rooms
+                        .roomid_shortroomid
+                        .get(&room_id)
+                        .unwrap()
+                        .expect("shortroomid should exist");
+
+                    let mut new_key = short_room_id;
+                    new_key.extend_from_slice(count);
+
+                    Some((new_key, v))
+                });
+
+                db.rooms.pduid_pdu.insert_batch(&mut batch)?;
+
+                let mut batch2 = db.rooms.eventid_pduid.iter().filter_map(|(k, value)| {
+                    if !value.starts_with(b"!") {
+                        return None;
+                    }
+                    let mut parts = value.splitn(2, |&b| b == 0xff);
+                    let room_id = parts.next().unwrap();
+                    let count = parts.next().unwrap();
+
+                    let short_room_id = db
+                        .rooms
+                        .roomid_shortroomid
+                        .get(&room_id)
+                        .unwrap()
+                        .expect("shortroomid should exist");
+
+                    let mut new_value = short_room_id;
+                    new_value.extend_from_slice(count);
+
+                    Some((k, new_value))
+                });
+
+                db.rooms.eventid_pduid.insert_batch(&mut batch2)?;
+
+                db.globals.bump_database_version(8)?;
+
+                println!("Migration: 7 -> 8 finished");
+            }
+
+            if db.globals.database_version()? < 9 {
+                // Update tokenids db layout
+                let mut iter = db
+                    .rooms
+                    .tokenids
+                    .iter()
+                    .filter_map(|(key, _)| {
+                        if !key.starts_with(b"!") {
+                            return None;
+                        }
+                        let mut parts = key.splitn(4, |&b| b == 0xff);
+                        let room_id = parts.next().unwrap();
+                        let word = parts.next().unwrap();
+                        let _pdu_id_room = parts.next().unwrap();
+                        let pdu_id_count = parts.next().unwrap();
+
+                        let short_room_id = db
+                            .rooms
+                            .roomid_shortroomid
+                            .get(&room_id)
+                            .unwrap()
+                            .expect("shortroomid should exist");
+                        let mut new_key = short_room_id;
+                        new_key.extend_from_slice(word);
+                        new_key.push(0xff);
+                        new_key.extend_from_slice(pdu_id_count);
+                        println!("old {:?}", key);
+                        println!("new {:?}", new_key);
+                        Some((new_key, Vec::new()))
+                    })
+                    .peekable();
+
+                while iter.peek().is_some() {
+                    db.rooms
+                        .tokenids
+                        .insert_batch(&mut iter.by_ref().take(1000))?;
+                    println!("smaller batch done");
+                }
+
+                println!("Deleting starts");
+
+                let batch2 = db
+                    .rooms
+                    .tokenids
+                    .iter()
+                    .filter_map(|(key, _)| {
+                        if key.starts_with(b"!") {
+                            println!("del {:?}", key);
+                            Some(key)
+                        } else {
+                            None
+                        }
+                    })
+                    .collect::<Vec<_>>();
+
+                for key in batch2 {
+                    println!("del");
+                    db.rooms.tokenids.remove(&key)?;
+                }
+
+                db.globals.bump_database_version(9)?;
+
+                println!("Migration: 8 -> 9 finished");
+            }
+
+            if db.globals.database_version()? < 10 {
+                // Add other direction for shortstatekeys
+                for (statekey, shortstatekey) in db.rooms.statekey_shortstatekey.iter() {
+                    db.rooms
+                        .shortstatekey_statekey
+                        .insert(&shortstatekey, &statekey)?;
+                }
+
+                // Force E2EE device list updates so we can send them over federation
+                for user_id in db.users.iter().filter_map(|r| r.ok()) {
+                    db.users
+                        .mark_device_key_update(&user_id, &db.rooms, &db.globals)?;
+                }
+
+                db.globals.bump_database_version(10)?;
+
+                println!("Migration: 9 -> 10 finished");
+            }
         }
 
         let guard = db.read().await;

src/database/abstraction.rs

@@ -35,6 +35,7 @@ pub trait Tree: Send + Sync {
     ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;
 
     fn increment(&self, key: &[u8]) -> Result<Vec<u8>>;
+    fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()>;
 
     fn scan_prefix<'a>(
         &'a self,

src/database/abstraction/sqlite.rs

@@ -9,15 +9,14 @@ use std::{
     path::{Path, PathBuf},
     pin::Pin,
     sync::Arc,
-    time::{Duration, Instant},
 };
+use thread_local::ThreadLocal;
 use tokio::sync::oneshot::Sender;
-use tracing::{debug, warn};
+use tracing::debug;
-
-pub const MILLI: Duration = Duration::from_millis(1);
 
 thread_local! {
     static READ_CONNECTION: RefCell<Option<&'static Connection>> = RefCell::new(None);
+    static READ_CONNECTION_ITERATOR: RefCell<Option<&'static Connection>> = RefCell::new(None);
 }
 
 struct PreparedStatementIterator<'a> {
@@ -42,6 +41,8 @@ impl<T> Drop for NonAliasingBox<T> {
 
 pub struct Engine {
     writer: Mutex<Connection>,
+    read_conn_tls: ThreadLocal<Connection>,
+    read_iterator_conn_tls: ThreadLocal<Connection>,
 
     path: PathBuf,
     cache_size_per_thread: u32,
@@ -51,11 +52,11 @@ impl Engine {
     fn prepare_conn(path: &Path, cache_size_kb: u32) -> Result<Connection> {
         let conn = Connection::open(&path)?;
 
-        conn.pragma_update(Some(Main), "page_size", &32768)?;
+        conn.pragma_update(Some(Main), "page_size", &2048)?;
         conn.pragma_update(Some(Main), "journal_mode", &"WAL")?;
         conn.pragma_update(Some(Main), "synchronous", &"NORMAL")?;
         conn.pragma_update(Some(Main), "cache_size", &(-i64::from(cache_size_kb)))?;
-        conn.pragma_update(Some(Main), "wal_autocheckpoint", &0)?;
+        conn.pragma_update(Some(Main), "wal_autocheckpoint", &2000)?;
 
         Ok(conn)
     }
@@ -64,19 +65,14 @@ impl Engine {
         self.writer.lock()
     }
 
-    fn read_lock(&self) -> &'static Connection {
+    fn read_lock(&self) -> &Connection {
-        READ_CONNECTION.with(|cell| {
+        self.read_conn_tls
-            let connection = &mut cell.borrow_mut();
+            .get_or(|| Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap())
-
-            if (*connection).is_none() {
-                let c = Box::leak(Box::new(
-                    Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap(),
-                ));
-                **connection = Some(c);
     }
 
-            connection.unwrap()
+    fn read_lock_iterator(&self) -> &Connection {
-        })
+        self.read_iterator_conn_tls
+            .get_or(|| Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap())
     }
 
     pub fn flush_wal(self: &Arc<Self>) -> Result<()> {
@@ -92,15 +88,18 @@ impl DatabaseEngine for Engine {
 
         // calculates cache-size per permanent connection
         // 1. convert MB to KiB
-        // 2. divide by permanent connections
+        // 2. divide by permanent connections + permanent iter connections + write connection
         // 3. round down to nearest integer
-        let cache_size_per_thread: u32 =
+        let cache_size_per_thread: u32 = ((config.db_cache_capacity_mb * 1024.0)
-            ((config.db_cache_capacity_mb * 1024.0) / (num_cpus::get().max(1) + 1) as f64) as u32;
+            / ((num_cpus::get().max(1) * 2) + 1) as f64)
+            as u32;
 
         let writer = Mutex::new(Self::prepare_conn(&path, cache_size_per_thread)?);
 
         let arc = Arc::new(Engine {
             writer,
+            read_conn_tls: ThreadLocal::new(),
+            read_iterator_conn_tls: ThreadLocal::new(),
             path,
             cache_size_per_thread,
         });
@@ -135,6 +134,7 @@ type TupleOfBytes = (Vec<u8>, Vec<u8>);
 impl SqliteTable {
     #[tracing::instrument(skip(self, guard, key))]
     fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result<Option<Vec<u8>>> {
+        //dbg!(&self.name);
         Ok(guard
             .prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())?
             .query_row([key], |row| row.get(0))
@@ -143,6 +143,7 @@ impl SqliteTable {
 
     #[tracing::instrument(skip(self, guard, key, value))]
     fn insert_with_guard(&self, guard: &Connection, key: &[u8], value: &[u8]) -> Result<()> {
+        //dbg!(&self.name);
         guard.execute(
             format!(
                 "INSERT OR REPLACE INTO {} (key, value) VALUES (?, ?)",
@@ -153,6 +154,39 @@ impl SqliteTable {
         )?;
         Ok(())
     }
+
+    pub fn iter_with_guard<'a>(
+        &'a self,
+        guard: &'a Connection,
+    ) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
+        let statement = Box::leak(Box::new(
+            guard
+                .prepare(&format!(
+                    "SELECT key, value FROM {} ORDER BY key ASC",
+                    &self.name
+                ))
+                .unwrap(),
+        ));
+
+        let statement_ref = NonAliasingBox(statement);
+
+        //let name = self.name.clone();
+
+        let iterator = Box::new(
+            statement
+                .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
+                .unwrap()
+                .map(move |r| {
+                    //dbg!(&name);
+                    r.unwrap()
+                }),
+        );
+
+        Box::new(PreparedStatementIterator {
+            iterator,
+            statement_ref,
+        })
+    }
 }
 
 impl Tree for SqliteTable {
@@ -164,16 +198,7 @@ impl Tree for SqliteTable {
     #[tracing::instrument(skip(self, key, value))]
     fn insert(&self, key: &[u8], value: &[u8]) -> Result<()> {
         let guard = self.engine.write_lock();
-
-        let start = Instant::now();
-
         self.insert_with_guard(&guard, key, value)?;
-
-        let elapsed = start.elapsed();
-        if elapsed > MILLI {
-            warn!("insert took {:?} : {}", elapsed, &self.name);
-        }
-
         drop(guard);
 
         let watchers = self.watchers.read();
@@ -216,53 +241,41 @@ impl Tree for SqliteTable {
         Ok(())
     }
 
+    #[tracing::instrument(skip(self, iter))]
+    fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
+        let guard = self.engine.write_lock();
+
+        guard.execute("BEGIN", [])?;
+        for key in iter {
+            let old = self.get_with_guard(&guard, &key)?;
+            let new = crate::utils::increment(old.as_deref())
+                .expect("utils::increment always returns Some");
+            self.insert_with_guard(&guard, &key, &new)?;
+        }
+        guard.execute("COMMIT", [])?;
+
+        drop(guard);
+
+        Ok(())
+    }
+
     #[tracing::instrument(skip(self, key))]
     fn remove(&self, key: &[u8]) -> Result<()> {
         let guard = self.engine.write_lock();
 
-        let start = Instant::now();
-
         guard.execute(
             format!("DELETE FROM {} WHERE key = ?", self.name).as_str(),
             [key],
         )?;
 
-        let elapsed = start.elapsed();
-
-        if elapsed > MILLI {
-            debug!("remove:    took {:012?} : {}", elapsed, &self.name);
-        }
-        // debug!("remove key: {:?}", &key);
-
         Ok(())
     }
 
     #[tracing::instrument(skip(self))]
     fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
-        let guard = self.engine.read_lock();
+        let guard = self.engine.read_lock_iterator();
 
-        let statement = Box::leak(Box::new(
+        self.iter_with_guard(&guard)
-            guard
-                .prepare(&format!(
-                    "SELECT key, value FROM {} ORDER BY key ASC",
-                    &self.name
-                ))
-                .unwrap(),
-        ));
-
-        let statement_ref = NonAliasingBox(statement);
-
-        let iterator = Box::new(
-            statement
-                .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
-                .unwrap()
-                .map(|r| r.unwrap()),
-        );
-
-        Box::new(PreparedStatementIterator {
-            iterator,
-            statement_ref,
-        })
     }
 
     #[tracing::instrument(skip(self, from, backwards))]
@@ -271,9 +284,11 @@ impl Tree for SqliteTable {
         from: &[u8],
         backwards: bool,
     ) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
-        let guard = self.engine.read_lock();
+        let guard = self.engine.read_lock_iterator();
         let from = from.to_vec(); // TODO change interface?
 
+        //let name = self.name.clone();
+
         if backwards {
             let statement = Box::leak(Box::new(
                 guard
@@ -290,7 +305,10 @@ impl Tree for SqliteTable {
                 statement
                     .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
                     .unwrap()
-                    .map(|r| r.unwrap()),
+                    .map(move |r| {
+                        //dbg!(&name);
+                        r.unwrap()
+                    }),
             );
             Box::new(PreparedStatementIterator {
                 iterator,
@@ -312,7 +330,10 @@ impl Tree for SqliteTable {
                 statement
                     .query_map([from], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
                     .unwrap()
-                    .map(|r| r.unwrap()),
+                    .map(move |r| {
+                        //dbg!(&name);
+                        r.unwrap()
+                    }),
             );
 
             Box::new(PreparedStatementIterator {
@@ -326,8 +347,6 @@ impl Tree for SqliteTable {
     fn increment(&self, key: &[u8]) -> Result<Vec<u8>> {
         let guard = self.engine.write_lock();
 
-        let start = Instant::now();
-
         let old = self.get_with_guard(&guard, key)?;
 
         let new =
@@ -335,26 +354,11 @@ impl Tree for SqliteTable {
 
         self.insert_with_guard(&guard, key, &new)?;
 
-        let elapsed = start.elapsed();
-
-        if elapsed > MILLI {
-            debug!("increment: took {:012?} : {}", elapsed, &self.name);
-        }
-        // debug!("increment key: {:?}", &key);
-
         Ok(new)
     }
 
     #[tracing::instrument(skip(self, prefix))]
     fn scan_prefix<'a>(&'a self, prefix: Vec<u8>) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
-        // let name = self.name.clone();
-        // self.iter_from_thread(
-        //     format!(
-        //         "SELECT key, value FROM {} WHERE key BETWEEN ?1 AND ?1 || X'FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF' ORDER BY key ASC",
-        //         name
-        //     )
-        //     [prefix]
-        // )
         Box::new(
             self.iter_from(&prefix, false)
                 .take_while(move |(key, _)| key.starts_with(&prefix)),

src/database/account_data.rs

@@ -40,7 +40,7 @@ impl AccountData {
         roomuserdataid.push(0xff);
         roomuserdataid.extend_from_slice(&event_type.as_bytes());
 
-        let mut key = prefix.clone();
+        let mut key = prefix;
         key.extend_from_slice(event_type.as_bytes());
 
         let json = serde_json::to_value(data).expect("all types here can be serialized"); // TODO: maybe add error handling

src/database/globals.rs

@@ -1,4 +1,4 @@
-use crate::{database::Config, utils, ConduitResult, Error, Result};
+use crate::{database::Config, server_server::FedDest, utils, ConduitResult, Error, Result};
 use ruma::{
     api::{
         client::r0::sync::sync_events,
@@ -6,25 +6,25 @@ use ruma::{
     },
     DeviceId, EventId, MilliSecondsSinceUnixEpoch, RoomId, ServerName, ServerSigningKeyId, UserId,
 };
-use rustls::{ServerCertVerifier, WebPKIVerifier};
 use std::{
     collections::{BTreeMap, HashMap},
     fs,
     future::Future,
+    net::IpAddr,
     path::PathBuf,
     sync::{Arc, Mutex, RwLock},
     time::{Duration, Instant},
 };
 use tokio::sync::{broadcast, watch::Receiver, Mutex as TokioMutex, Semaphore};
-use tracing::{error, info};
+use tracing::error;
 use trust_dns_resolver::TokioAsyncResolver;
 
 use super::abstraction::Tree;
 
 pub const COUNTER: &[u8] = b"c";
 
-type WellKnownMap = HashMap<Box<ServerName>, (String, String)>;
+type WellKnownMap = HashMap<Box<ServerName>, (FedDest, String)>;
-type TlsNameMap = HashMap<String, webpki::DNSName>;
+type TlsNameMap = HashMap<String, (Vec<IpAddr>, u16)>;
 type RateLimitState = (Instant, u32); // Time if last failed try, number of failed tries
 type SyncHandle = (
     Option<String>,                                         // since
@@ -37,7 +37,6 @@ pub struct Globals {
     pub(super) globals: Arc<dyn Tree>,
     config: Config,
     keypair: Arc<ruma::signatures::Ed25519KeyPair>,
-    reqwest_client: reqwest::Client,
     dns_resolver: TokioAsyncResolver,
     jwt_decoding_key: Option<jsonwebtoken::DecodingKey<'static>>,
     pub(super) server_signingkeys: Arc<dyn Tree>,
@@ -51,40 +50,6 @@ pub struct Globals {
     pub rotate: RotationHandler,
 }
 
-struct MatrixServerVerifier {
-    inner: WebPKIVerifier,
-    tls_name_override: Arc<RwLock<TlsNameMap>>,
-}
-
-impl ServerCertVerifier for MatrixServerVerifier {
-    #[tracing::instrument(skip(self, roots, presented_certs, dns_name, ocsp_response))]
-    fn verify_server_cert(
-        &self,
-        roots: &rustls::RootCertStore,
-        presented_certs: &[rustls::Certificate],
-        dns_name: webpki::DNSNameRef<'_>,
-        ocsp_response: &[u8],
-    ) -> std::result::Result<rustls::ServerCertVerified, rustls::TLSError> {
-        if let Some(override_name) = self.tls_name_override.read().unwrap().get(dns_name.into()) {
-            let result = self.inner.verify_server_cert(
-                roots,
-                presented_certs,
-                override_name.as_ref(),
-                ocsp_response,
-            );
-            if result.is_ok() {
-                return result;
-            }
-            info!(
-                "Server {:?} is non-compliant, retrying TLS verification with original name",
-                dns_name
-            );
-        }
-        self.inner
-            .verify_server_cert(roots, presented_certs, dns_name, ocsp_response)
-    }
-}
-
 /// Handles "rotation" of long-polling requests. "Rotation" in this context is similar to "rotation" of log files and the like.
 ///
 /// This is utilized to have sync workers return early and release read locks on the database.
@@ -162,24 +127,6 @@ impl Globals {
         };
 
         let tls_name_override = Arc::new(RwLock::new(TlsNameMap::new()));
-        let verifier = Arc::new(MatrixServerVerifier {
-            inner: WebPKIVerifier::new(),
-            tls_name_override: tls_name_override.clone(),
-        });
-        let mut tlsconfig = rustls::ClientConfig::new();
-        tlsconfig.dangerous().set_certificate_verifier(verifier);
-        tlsconfig.root_store =
-            rustls_native_certs::load_native_certs().expect("Error loading system certificates");
-
-        let mut reqwest_client_builder = reqwest::Client::builder()
-            .connect_timeout(Duration::from_secs(30))
-            .timeout(Duration::from_secs(60 * 3))
-            .pool_max_idle_per_host(1)
-            .use_preconfigured_tls(tlsconfig);
-        if let Some(proxy) = config.proxy.to_proxy()? {
-            reqwest_client_builder = reqwest_client_builder.proxy(proxy);
-        }
-        let reqwest_client = reqwest_client_builder.build().unwrap();
 
         let jwt_decoding_key = config
             .jwt_secret
@@ -190,7 +137,6 @@ impl Globals {
             globals,
             config,
             keypair: Arc::new(keypair),
-            reqwest_client,
             dns_resolver: TokioAsyncResolver::tokio_from_system_conf().map_err(|_| {
                 Error::bad_config("Failed to set up trust dns resolver with system config.")
             })?,
@@ -219,8 +165,16 @@ impl Globals {
     }
 
     /// Returns a reqwest client which can be used to send requests.
-    pub fn reqwest_client(&self) -> &reqwest::Client {
+    pub fn reqwest_client(&self) -> Result<reqwest::ClientBuilder> {
-        &self.reqwest_client
+        let mut reqwest_client_builder = reqwest::Client::builder()
+            .connect_timeout(Duration::from_secs(30))
+            .timeout(Duration::from_secs(60 * 3))
+            .pool_max_idle_per_host(1);
+        if let Some(proxy) = self.config.proxy.to_proxy()? {
+            reqwest_client_builder = reqwest_client_builder.proxy(proxy);
+        }
+
+        Ok(reqwest_client_builder)
     }
 
     #[tracing::instrument(skip(self))]
@@ -273,7 +227,11 @@ impl Globals {
     /// Remove the outdated keys and insert the new ones.
     ///
     /// This doesn't actually check that the keys provided are newer than the old set.
-    pub fn add_signing_key(&self, origin: &ServerName, new_keys: ServerSigningKeys) -> Result<()> {
+    pub fn add_signing_key(
+        &self,
+        origin: &ServerName,
+        new_keys: ServerSigningKeys,
+    ) -> Result<BTreeMap<ServerSigningKeyId, VerifyKey>> {
         // Not atomic, but this is not critical
         let signingkeys = self.server_signingkeys.get(origin.as_bytes())?;
 
@@ -298,7 +256,14 @@ impl Globals {
             &serde_json::to_vec(&keys).expect("serversigningkeys can be serialized"),
         )?;
 
-        Ok(())
+        let mut tree = keys.verify_keys;
+        tree.extend(
+            keys.old_verify_keys
+                .into_iter()
+                .map(|old| (old.0, VerifyKey::new(old.1.key))),
+        );
+
+        Ok(tree)
     }
 
     /// This returns an empty `Ok(BTreeMap<..>)` when there are no keys found for the server.

src/database/key_backups.rs

@@ -84,6 +84,27 @@ impl KeyBackups {
         Ok(version.to_string())
     }
 
+    pub fn get_latest_backup_version(&self, user_id: &UserId) -> Result<Option<String>> {
+        let mut prefix = user_id.as_bytes().to_vec();
+        prefix.push(0xff);
+        let mut last_possible_key = prefix.clone();
+        last_possible_key.extend_from_slice(&u64::MAX.to_be_bytes());
+
+        self.backupid_algorithm
+            .iter_from(&last_possible_key, true)
+            .take_while(move |(k, _)| k.starts_with(&prefix))
+            .next()
+            .map_or(Ok(None), |(key, _)| {
+                utils::string_from_bytes(
+                    key.rsplit(|&b| b == 0xff)
+                        .next()
+                        .expect("rsplit always returns an element"),
+                )
+                .map_err(|_| Error::bad_database("backupid_algorithm key is invalid."))
+                .map(Some)
+            })
+    }
+
     pub fn get_latest_backup(&self, user_id: &UserId) -> Result<Option<(String, BackupAlgorithm)>> {
         let mut prefix = user_id.as_bytes().to_vec();
         prefix.push(0xff);

src/database/pusher.rs

@@ -113,7 +113,11 @@ where
     //*reqwest_request.timeout_mut() = Some(Duration::from_secs(5));
 
     let url = reqwest_request.url().clone();
-    let response = globals.reqwest_client().execute(reqwest_request).await;
+    let response = globals
+        .reqwest_client()?
+        .build()?
+        .execute(reqwest_request)
+        .await;
 
     match response {
         Ok(mut response) => {

src/database/rooms/edus.rs

@@ -422,7 +422,7 @@ impl RoomEdus {
     }
 
     /// Sets all users to offline who have been quiet for too long.
-    fn presence_maintain(
+    fn _presence_maintain(
         &self,
         rooms: &super::Rooms,
         globals: &super::super::globals::Globals,
@@ -489,13 +489,13 @@ impl RoomEdus {
     }
 
     /// Returns an iterator over the most recent presence updates that happened after the event with id `since`.
-    #[tracing::instrument(skip(self, globals, rooms))]
+    #[tracing::instrument(skip(self, since, _rooms, _globals))]
     pub fn presence_since(
         &self,
         room_id: &RoomId,
         since: u64,
-        rooms: &super::Rooms,
+        _rooms: &super::Rooms,
-        globals: &super::super::globals::Globals,
+        _globals: &super::super::globals::Globals,
     ) -> Result<HashMap<UserId, PresenceEvent>> {
         //self.presence_maintain(rooms, globals)?;
 

src/database/sending.rs

@@ -1,5 +1,5 @@
 use std::{
-    collections::{BTreeMap, HashMap},
+    collections::{BTreeMap, HashMap, HashSet},
     convert::{TryFrom, TryInto},
     fmt::Debug,
     sync::Arc,
@@ -20,14 +20,17 @@ use ruma::{
         appservice,
         federation::{
             self,
-            transactions::edu::{Edu, ReceiptContent, ReceiptData, ReceiptMap},
+            transactions::edu::{
+                DeviceListUpdateContent, Edu, ReceiptContent, ReceiptData, ReceiptMap,
+            },
         },
         OutgoingRequest,
     },
+    device_id,
     events::{push_rules, AnySyncEphemeralRoomEvent, EventType},
     push,
     receipt::ReceiptType,
-    MilliSecondsSinceUnixEpoch, ServerName, UInt, UserId,
+    uint, MilliSecondsSinceUnixEpoch, ServerName, UInt, UserId,
 };
 use tokio::{
     select,
@@ -81,8 +84,8 @@ pub enum SendingEventType {
 pub struct Sending {
     /// The state for a given state hash.
     pub(super) servername_educount: Arc<dyn Tree>, // EduCount: Count of last EDU sync
-    pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / * (for edus), Data = EDU content
+    pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / Id (for edus), Data = EDU content
-    pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / * (for edus), Data = EDU content
+    pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / Id (for edus), Data = EDU content
     pub(super) maximum_requests: Arc<Semaphore>,
     pub sender: mpsc::UnboundedSender<(Vec<u8>, Vec<u8>)>,
 }
@@ -317,8 +320,19 @@ impl Sending {
             })?;
         let mut events = Vec::new();
         let mut max_edu_count = since;
+        let mut device_list_changes = HashSet::new();
+
         'outer: for room_id in db.rooms.server_rooms(server) {
             let room_id = room_id?;
+            // Look for device list updates in this room
+            device_list_changes.extend(
+                db.users
+                    .keys_changed(&room_id.to_string(), since, None)
+                    .filter_map(|r| r.ok())
+                    .filter(|user_id| user_id.server_name() == db.globals.server_name()),
+            );
+
+            // Look for read receipts in this room
             for r in db.rooms.edus.readreceipts_since(&room_id, since) {
                 let (user_id, count, read_receipt) = r?;
 
@@ -378,6 +392,22 @@ impl Sending {
             }
         }
 
+        for user_id in device_list_changes {
+            // Empty prev id forces synapse to resync: https://github.com/matrix-org/synapse/blob/98aec1cc9da2bd6b8e34ffb282c85abf9b8b42ca/synapse/handlers/device.py#L767
+            // Because synapse resyncs, we can just insert dummy data
+            let edu = Edu::DeviceListUpdate(DeviceListUpdateContent {
+                user_id,
+                device_id: device_id!("dummy"),
+                device_display_name: "Dummy".to_owned(),
+                stream_id: uint!(1),
+                prev_id: Vec::new(),
+                deleted: None,
+                keys: None,
+            });
+
+            events.push(serde_json::to_vec(&edu).expect("json can be serialized"));
+        }
+
         Ok((events, max_edu_count))
     }
 
@@ -405,10 +435,15 @@ impl Sending {
     }
 
     #[tracing::instrument(skip(self, server, serialized))]
-    pub fn send_reliable_edu(&self, server: &ServerName, serialized: Vec<u8>) -> Result<()> {
+    pub fn send_reliable_edu(
+        &self,
+        server: &ServerName,
+        serialized: Vec<u8>,
+        id: u64,
+    ) -> Result<()> {
         let mut key = server.as_bytes().to_vec();
         key.push(0xff);
-        key.push(b'*');
+        key.extend_from_slice(&id.to_be_bytes());
         self.servernameevent_data.insert(&key, &serialized)?;
         self.sender.unbounded_send((key, serialized)).unwrap();
 
@@ -684,10 +719,10 @@ impl Sending {
                 OutgoingKind::Appservice(Box::<ServerName>::try_from(server).map_err(|_| {
                     Error::bad_database("Invalid server string in server_currenttransaction")
                 })?),
-                if event.starts_with(b"*") {
+                if value.is_empty() {
-                    SendingEventType::Edu(value)
-                } else {
                     SendingEventType::Pdu(event.to_vec())
+                } else {
+                    SendingEventType::Edu(value)
                 },
             )
         } else if key.starts_with(b"$") {
@@ -702,10 +737,10 @@ impl Sending {
                 .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
             (
                 OutgoingKind::Push(user.to_vec(), pushkey.to_vec()),
-                if event.starts_with(b"*") {
+                if value.is_empty() {
-                    SendingEventType::Edu(value)
-                } else {
                     SendingEventType::Pdu(event.to_vec())
+                } else {
+                    SendingEventType::Edu(value)
                 },
             )
         } else {
@@ -723,10 +758,10 @@ impl Sending {
                 OutgoingKind::Normal(Box::<ServerName>::try_from(server).map_err(|_| {
                     Error::bad_database("Invalid server string in server_currenttransaction")
                 })?),
-                if event.starts_with(b"*") {
+                if value.is_empty() {
-                    SendingEventType::Edu(event[1..].to_vec())
-                } else {
                     SendingEventType::Pdu(event.to_vec())
+                } else {
+                    SendingEventType::Edu(value)
                 },
             )
         })

src/database/uiaa.rs

@@ -4,11 +4,14 @@ use crate::{client_server::SESSION_ID_LENGTH, utils, Error, Result};
 use ruma::{
     api::client::{
         error::ErrorKind,
-        r0::uiaa::{IncomingAuthData, UiaaInfo},
+        r0::uiaa::{
+            IncomingAuthData, IncomingPassword, IncomingUserIdentifier::MatrixId, UiaaInfo,
+        },
     },
     signatures::CanonicalJsonValue,
     DeviceId, UserId,
 };
+use tracing::error;
 
 use super::abstraction::Tree;
 
@@ -49,14 +52,8 @@ impl Uiaa {
         users: &super::users::Users,
         globals: &super::globals::Globals,
     ) -> Result<(bool, UiaaInfo)> {
-        if let IncomingAuthData::DirectRequest {
+        let mut uiaainfo = auth
-            kind,
+            .session()
-            session,
-            auth_parameters,
-        } = &auth
-        {
-            let mut uiaainfo = session
-                .as_ref()
             .map(|session| self.get_uiaa_session(&user_id, &device_id, session))
             .unwrap_or_else(|| Ok(uiaainfo.clone()))?;
 
@@ -64,55 +61,29 @@ impl Uiaa {
             uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
         }
 
+        match auth {
             // Find out what the user completed
-            match &**kind {
+            IncomingAuthData::Password(IncomingPassword {
-                "m.login.password" => {
+                identifier,
-                    let identifier = auth_parameters.get("identifier").ok_or(Error::BadRequest(
+                password,
-                        ErrorKind::MissingParam,
+                ..
-                        "m.login.password needs identifier.",
+            }) => {
-                    ))?;
+                let username = match identifier {
-
+                    MatrixId(username) => username,
-                    let identifier_type = identifier.get("type").ok_or(Error::BadRequest(
+                    _ => {
-                        ErrorKind::MissingParam,
-                        "Identifier needs a type.",
-                    ))?;
-
-                    if identifier_type != "m.id.user" {
                         return Err(Error::BadRequest(
                             ErrorKind::Unrecognized,
                             "Identifier type not recognized.",
-                        ));
+                        ))
                     }
+                };
 
-                    let username = identifier
+                let user_id =
-                        .get("user")
+                    UserId::parse_with_server_name(username.clone(), globals.server_name())
-                        .ok_or(Error::BadRequest(
-                            ErrorKind::MissingParam,
-                            "Identifier needs user field.",
-                        ))?
-                        .as_str()
-                        .ok_or(Error::BadRequest(
-                            ErrorKind::BadJson,
-                            "User is not a string.",
-                        ))?;
-
-                    let user_id = UserId::parse_with_server_name(username, globals.server_name())
                         .map_err(|_| {
                             Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid.")
                         })?;
 
-                    let password = auth_parameters
-                        .get("password")
-                        .ok_or(Error::BadRequest(
-                            ErrorKind::MissingParam,
-                            "Password is missing.",
-                        ))?
-                        .as_str()
-                        .ok_or(Error::BadRequest(
-                            ErrorKind::BadJson,
-                            "Password is not a string.",
-                        ))?;
-
                 // Check if password is correct
                 if let Some(hash) = users.password_hash(&user_id)? {
                     let hash_matches =
@@ -130,10 +101,10 @@ impl Uiaa {
                 // Password was correct! Let's add it to `completed`
                 uiaainfo.completed.push("m.login.password".to_owned());
             }
-                "m.login.dummy" => {
+            IncomingAuthData::Dummy(_) => {
                 uiaainfo.completed.push("m.login.dummy".to_owned());
             }
-                k => panic!("type not supported: {}", k),
+            k => error!("type not supported: {:?}", k),
         }
 
         // Check if a flow now succeeds
@@ -166,9 +137,6 @@ impl Uiaa {
             None,
         )?;
         Ok((true, uiaainfo))
-        } else {
-            panic!("FallbackAcknowledgement is not supported yet");
-        }
     }
 
     fn set_uiaa_request(

src/database/users.rs

@@ -673,7 +673,7 @@ impl Users {
     }
 
     #[tracing::instrument(skip(self, user_id, rooms, globals))]
-    fn mark_device_key_update(
+    pub fn mark_device_key_update(
         &self,
         user_id: &UserId,
         rooms: &super::rooms::Rooms,

src/main.rs

@@ -17,7 +17,7 @@ use std::sync::Arc;
 use database::Config;
 pub use database::Database;
 pub use error::{Error, Result};
-use opentelemetry::trace::Tracer;
+use opentelemetry::trace::{FutureExt, Tracer};
 pub use pdu::PduEvent;
 pub use rocket::State;
 use ruma::api::client::error::ErrorKind;
@@ -160,7 +160,8 @@ fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<
                 server_server::get_room_state_route,
                 server_server::get_room_state_ids_route,
                 server_server::create_join_event_template_route,
-                server_server::create_join_event_route,
+                server_server::create_join_event_v1_route,
+                server_server::create_join_event_v2_route,
                 server_server::create_invite_route,
                 server_server::get_devices_route,
                 server_server::get_room_information_route,
@@ -220,14 +221,17 @@ async fn main() {
     };
 
     if config.allow_jaeger {
+        opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
         let tracer = opentelemetry_jaeger::new_pipeline()
-            .with_service_name("conduit")
+            .install_batch(opentelemetry::runtime::Tokio)
-            .install_simple()
             .unwrap();
 
         let span = tracer.start("conduit");
-        start.await;
+        start.with_current_context().await;
         drop(span);
+
+        println!("exporting");
+        opentelemetry::global::shutdown_tracer_provider();
     } else {
         std::env::set_var("RUST_LOG", &config.log);
 

src/pdu.rs

@@ -12,7 +12,7 @@ use ruma::{
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom};
-use tracing::error;
+use tracing::warn;
 
 #[derive(Clone, Deserialize, Serialize, Debug)]
 pub struct PduEvent {
@@ -322,7 +322,7 @@ pub(crate) fn gen_event_id_canonical_json(
     pdu: &Raw<ruma::events::pdu::Pdu>,
 ) -> crate::Result<(EventId, CanonicalJsonObject)> {
     let value = serde_json::from_str(pdu.json().get()).map_err(|e| {
-        error!("{:?}: {:?}", pdu, e);
+        warn!("Error parsing incoming event {:?}: {:?}", pdu, e);
         Error::BadServerResponse("Invalid PDU in server response")
     })?;
 

src/ruma_wrapper.rs

@@ -66,7 +66,11 @@ where
         let limit = db.globals.max_request_size();
         let mut handle = data.open(ByteUnit::Byte(limit.into()));
         let mut body = Vec::new();
-        handle.read_to_end(&mut body).await.unwrap();
+        if let Err(_) = handle.read_to_end(&mut body).await {
+            // Client disconnected
+            // Missing Token
+            return Failure((Status::new(582), ()));
+        }
 
         let mut json_body = serde_json::from_slice::<CanonicalJsonValue>(&body).ok();