fix: long prev event fetch times for huge rooms

less warnings
improvement: cache for short event ids
2021-08-12 17:55:16 +02:00 · 2021-08-11 21:17:01 +02:00 · 2021-08-11 21:14:22 +02:00 · 2021-08-11 19:15:38 +02:00 · 2021-08-11 10:24:16 +02:00
23 changed files with 1319 additions and 2803 deletions
@@ -9,6 +9,85 @@ variables:
  FF_USE_FASTZIP: 1
  CACHE_COMPRESSION_LEVEL: fastest

+test:cargo:
+  stage: "test"
+  needs: []
+  image: "rust:latest"
+  tags: ["docker"]
+  variables:
+    CARGO_HOME: "cargohome"
+  cache:
+    paths:
+      - target
+      - cargohome
+    key: test_cache
+  interruptible: true
+  before_script:
+    - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
+    - apt-get update -yqq
+    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
+    - rustup component add clippy rustfmt
+  script:
+    - rustc --version && cargo --version  # Print version info for debugging
+    - cargo fmt --all -- --check
+    - cargo test --workspace --verbose --locked
+    - cargo clippy
+
+test:sytest:
+  stage: "test"
+  allow_failure: true
+  needs:
+    - "build:cargo:x86_64-unknown-linux-musl"
+  image:
+    name: "valkum/sytest-conduit:latest"
+    entrypoint: [""]
+  tags: ["docker"]
+  variables:
+    PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
+  before_script:
+    - "mkdir -p /app"
+    - "cp ./conduit-x86_64-unknown-linux-musl /app/conduit"
+    - "chmod +x /app/conduit"
+    - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
+    - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
+    - "cd /"
+  script:
+    - "SYTEST_EXIT_CODE=0"
+    - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
+    - "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap"
+    - "exit $SYTEST_EXIT_CODE"
+  artifacts:
+    when: always
+    paths:
+      - "$CI_PROJECT_DIR/sytest.xml"
+      - "$CI_PROJECT_DIR/results.tap"
+    reports:
+      junit: "$CI_PROJECT_DIR/sytest.xml"
+
+
+test:register:element-web-stable:
+  stage: "test"
+  needs:
+    - "build:cargo:x86_64-unknown-linux-gnu"
+  image: "buildkite/puppeteer:latest"
+  tags: ["docker"]
+  interruptible: true
+  script:
+    - "CONDUIT_CONFIG=tests/test-config.toml ./conduit-x86_64-unknown-linux-gnu > conduit.log &"
+    - "cd tests/client-element-web/"
+    - "npm install puppeteer"
+    - "node test-element-web-registration.js \"https://app.element.io/\" \"http://localhost:6167\""
+    - "killall --regexp \"conduit\""
+    - "cd ../.."
+    - "cat conduit.log"
+  artifacts:
+    paths:
+      - "tests/client-element-web/*.png"
+      - "*.log"
+    expire_in: 1 week
+    when: always
+  retry: 1
+

 # --------------------------------------------------------------------- #
 #  Cargo: Compiling for different architectures                         #
@@ -26,10 +105,7 @@ variables:
    paths:
      - cargohome
      - target/
-    key: "build_cache-$TARGET-release"
-  variables:
-    CARGO_PROFILE_RELEASE_LTO=true
-    CARGO_PROFILE_RELEASE_CODEGEN_UNITS=1
+    key: "build_cache-$TARGET"
  before_script:
    - 'echo "Building for target $TARGET"'
    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
@@ -39,21 +115,26 @@ variables:
    - "apt-get install -yqq --no-install-recommends $NEEDED_PACKAGES"
    - "rustup target add $TARGET"
  script:
+    # Set some cargo tuning here, because targets overwrite the 'variables'
+    - "export CARGO_INCREMENTAL=true"
+    - "export CARGO_PROFILE_RELEASE_CODEGEN_UNITS=16"
+    - "export CARGO_PROFILE_RELEASE_LTO=thin"
    - time cargo build --target $TARGET --release
    - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'

-
-build:release:cargo:x86_64-unknown-linux-gnu:
+build:cargo:x86_64-unknown-linux-gnu:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "x86_64-unknown-linux-gnu"
+  rules:
+    - if: "$CI_COMMIT_BRANCH"
  artifacts:
    name: "conduit-x86_64-unknown-linux-gnu"
    paths:
      - "conduit-x86_64-unknown-linux-gnu"
-    expose_as: "Conduit for x86_64-unknown-linux-gnu"
+    expose_as: "Release binary x86_64-unknown-linux-gnu"

-build:release:cargo:armv7-unknown-linux-gnueabihf:
+build:cargo:armv7-unknown-linux-gnueabihf:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "armv7-unknown-linux-gnueabihf"
@@ -65,9 +146,9 @@ build:release:cargo:armv7-unknown-linux-gnueabihf:
    name: "conduit-armv7-unknown-linux-gnueabihf"
    paths:
      - "conduit-armv7-unknown-linux-gnueabihf"
-    expose_as: "Conduit for armv7-unknown-linux-gnueabihf"
+    expose_as: "Release binary armv7-unknown-linux-gnueabihf"

-build:release:cargo:aarch64-unknown-linux-gnu:
+build:cargo:aarch64-unknown-linux-gnu:
  extends: .build-cargo-shared-settings
  variables:
    TARGET: "aarch64-unknown-linux-gnu"
@@ -81,11 +162,13 @@ build:release:cargo:aarch64-unknown-linux-gnu:
    name: "conduit-aarch64-unknown-linux-gnu"
    paths:
      - "conduit-aarch64-unknown-linux-gnu"
-    expose_as: "Conduit for aarch64-unknown-linux-gnu"
+    expose_as: "Release binary aarch64-unknown-linux-gnu"

-build:release:cargo:x86_64-unknown-linux-musl:
+build:cargo:x86_64-unknown-linux-musl:
  extends: .build-cargo-shared-settings
  image: "rust:alpine"
+  rules:
+    - if: '$CI_COMMIT_BRANCH' # Always run
  variables:
    TARGET: "x86_64-unknown-linux-musl"
  before_script:
@@ -98,47 +181,7 @@ build:release:cargo:x86_64-unknown-linux-musl:
    name: "conduit-x86_64-unknown-linux-musl"
    paths:
      - "conduit-x86_64-unknown-linux-musl"
-    expose_as: "Conduit for x86_64-unknown-linux-musl"
-
-
-
-.cargo-debug-shared-settings:
-  extends: ".build-cargo-shared-settings"
-  rules:
-    - if: '$CI_COMMIT_BRANCH'
-  cache:
-    key: "build_cache-$TARGET-debug"
-  script:
-    - "time cargo build --target $TARGET"
-    - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
-
-build:debug:cargo:x86_64-unknown-linux-gnu:
-  extends: ".cargo-debug-shared-settings"
-  variables:
-    TARGET: "x86_64-unknown-linux-gnu"
-  artifacts:
-    name: "conduit-debug-x86_64-unknown-linux-gnu"
-    paths:
-      - "conduit-debug-x86_64-unknown-linux-gnu"
-    expose_as: "Conduit DEBUG for x86_64-unknown-linux-gnu"
-
-build:debug:cargo:x86_64-unknown-linux-musl:
-  extends: ".cargo-debug-shared-settings"
-  image: "rust:alpine"
-  variables:
-    TARGET: "x86_64-unknown-linux-musl"
-  before_script:
-    - 'echo "Building for target $TARGET"'
-    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
-    - "cat /etc/*-release && rustc --version && cargo --version"  # Print version info for debugging
-    - "rustup target add $TARGET"
-    - "apk add libc-dev"
-  artifacts:
-    name: "conduit-debug-x86_64-unknown-linux-musl"
-    paths:
-      - "conduit-debug-x86_64-unknown-linux-musl"
-    expose_as: "Conduit DEBUG for x86_64-unknown-linux-musl"
-
+    expose_as: "Release binary x86_64-unknown-linux-musl"


 # --------------------------------------------------------------------- #
@@ -148,7 +191,7 @@ build:debug:cargo:x86_64-unknown-linux-musl:

 .build-cargo-deb-shared-settings:
  stage: "build"
-  needs: [ ]
+  needs: []
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
  interruptible: true
@@ -209,7 +252,7 @@ build:cargo-deb:x86_64-unknown-linux-gnu:
 build:docker:main:
  extends: .docker-shared-settings
  needs:
-    - "build:release:cargo:x86_64-unknown-linux-musl"
+    - "build:cargo:x86_64-unknown-linux-musl"
  script:
    - >
      /kaniko/executor
@@ -229,91 +272,6 @@ build:docker:main:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'


-
-# --------------------------------------------------------------------- #
-#  Run tests                                                            #
-# --------------------------------------------------------------------- #
-
-test:cargo:
-  stage: "test"
-  needs: [ ]
-  image: "rust:latest"
-  tags: [ "docker" ]
-  variables:
-    CARGO_HOME: "cargohome"
-  cache:
-    paths:
-      - target
-      - cargohome
-    key: test_cache
-  interruptible: true
-  before_script:
-    - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
-    - apt-get update -yqq
-    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
-    - rustup component add clippy rustfmt
-  script:
-    - rustc --version && cargo --version  # Print version info for debugging
-    - cargo fmt --all -- --check
-    - cargo test --workspace --verbose --locked
-    - cargo clippy
-
-test:sytest:
-  stage: "test"
-  allow_failure: true
-  needs:
-    - "build:debug:cargo:x86_64-unknown-linux-musl"
-  image:
-    name: "valkum/sytest-conduit:latest"
-    entrypoint: [ "" ]
-  tags: [ "docker" ]
-  variables:
-    PLUGINS: "https://github.com/valkum/sytest_conduit/archive/master.tar.gz"
-  before_script:
-    - "mkdir -p /app"
-    - "cp ./conduit-debug-x86_64-unknown-linux-musl /app/conduit"
-    - "chmod +x /app/conduit"
-    - "rm -rf /src && ln -s $CI_PROJECT_DIR/ /src"
-    - "mkdir -p /work/server-0/database/ && mkdir -p /work/server-1/database/ && mkdir -p /work/server-2/database/"
-    - "cd /"
-  script:
-    - "SYTEST_EXIT_CODE=0"
-    - "/bootstrap.sh conduit || SYTEST_EXIT_CODE=1"
-    - "perl /sytest/tap-to-junit-xml.pl --puretap --input /logs/results.tap --output $CI_PROJECT_DIR/sytest.xml \"Sytest\" && cp /logs/results.tap $CI_PROJECT_DIR/results.tap"
-    - "exit $SYTEST_EXIT_CODE"
-  artifacts:
-    when: always
-    paths:
-      - "$CI_PROJECT_DIR/sytest.xml"
-      - "$CI_PROJECT_DIR/results.tap"
-    reports:
-      junit: "$CI_PROJECT_DIR/sytest.xml"
-
-
-test:register:element-web-stable:
-  stage: "test"
-  needs:
-    - "build:debug:cargo:x86_64-unknown-linux-gnu"
-  image: "buildkite/puppeteer:latest"
-  tags: [ "docker" ]
-  interruptible: true
-  script:
-    - "CONDUIT_CONFIG=tests/test-config.toml ./conduit-debug-x86_64-unknown-linux-gnu > conduit.log &"
-    - "cd tests/client-element-web/"
-    - "npm install puppeteer"
-    - "node test-element-web-registration.js \"https://app.element.io/\" \"http://localhost:6167\""
-    - "killall --regexp \"conduit\""
-    - "cd ../.."
-    - "cat conduit.log"
-  artifacts:
-    paths:
-      - "tests/client-element-web/*.png"
-      - "*.log"
-    expire_in: 1 week
-    when: always
-  retry: 1
-
-
 # --------------------------------------------------------------------- #
 #  Store binaries as package so they have download urls                 #
 # --------------------------------------------------------------------- #
@@ -321,10 +279,10 @@ test:register:element-web-stable:
 publish:package:
  stage: "upload artifacts"
  needs:
-    - "build:release:cargo:x86_64-unknown-linux-gnu"
-    - "build:release:cargo:armv7-unknown-linux-gnueabihf"
-    - "build:release:cargo:aarch64-unknown-linux-gnu"
-    - "build:release:cargo:x86_64-unknown-linux-musl"
+    - "build:cargo:x86_64-unknown-linux-gnu"
+    - "build:cargo:armv7-unknown-linux-gnueabihf"
+    - "build:cargo:aarch64-unknown-linux-gnu"
+    - "build:cargo:x86_64-unknown-linux-musl"
    - "build:cargo-deb:x86_64-unknown-linux-gnu"
  rules:
    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
@@ -146,9 +146,9 @@ dependencies = [

 [[package]]
 name = "bitflags"
-version = "1.3.2"
+version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+checksum = "cf1de2fe8c75bc145a2f577add951f8134889b4795d47466a54a5c846d691693"

 [[package]]
 name = "blake2b_simd"
@@ -190,9 +190,9 @@ checksum = "14c189c53d098945499cdfa7ecc63567cf3886b3332b312a5b4585d8d3a6a610"

 [[package]]
 name = "bytes"
-version = "1.1.0"
+version = "1.0.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c4872d67bab6358e59559027aa3b9157c53d9358c51423c17554809a8858e0f8"
+checksum = "b700ce4376041dcd0a327fd0097c41095743c4c8af8887265942faf1100bd040"

 [[package]]
 name = "cc"
@@ -248,7 +248,7 @@ dependencies = [
 "jsonwebtoken",
 "lru-cache",
 "num_cpus",
- "opentelemetry 0.16.0",
+ "opentelemetry",
 "opentelemetry-jaeger",
 "parking_lot",
 "pretty_env_logger",
@@ -295,6 +295,12 @@ version = "0.1.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "245097e9a4535ee1e3e3931fcfcd55a796a44c643e8596ff6566d68f09b87bbc"

+[[package]]
+name = "convert_case"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6245d59a3e82a7fc217c5828a6692dbc6dfb63a0c8c90495621f7b9d79704a0e"
+
 [[package]]
 name = "cookie"
 version = "0.15.1"
@@ -366,9 +372,9 @@ dependencies = [

 [[package]]
 name = "crossbeam-deque"
-version = "0.8.1"
+version = "0.8.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6455c0ca19f0d2fbf751b908d5c55c1f5cbc65e03c4225427254b46890bdde1e"
+checksum = "94af6efb46fef72616855b036a624cf27ba656ffc9be1b9a3c931cfc7749a9a9"
 dependencies = [
 "cfg-if 1.0.0",
 "crossbeam-epoch",
@@ -429,9 +435,9 @@ dependencies = [

 [[package]]
 name = "curve25519-dalek"
-version = "3.2.0"
+version = "3.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0b9fdf9972b2bd6af2d913799d9ebc165ea4d2e65878e329d9c6b372c4491b61"
+checksum = "639891fde0dbea823fc3d798a0fdf9d2f9440a42d64a78ab3488b0ca025117b3"
 dependencies = [
 "byteorder",
 "digest",
@@ -458,13 +464,26 @@ dependencies = [

 [[package]]
 name = "der"
-version = "0.4.1"
+version = "0.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "31e21d2d0f22cde6e88694108429775c0219760a07779bf96503b434a03d7412"
+checksum = "49f215f706081a44cb702c71c39a52c05da637822e9c1645a50b7202689e982d"
 dependencies = [
 "const-oid",
 ]

+[[package]]
+name = "derive_more"
+version = "0.99.16"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "40eebddd2156ce1bb37b20bbe5151340a31828b1f2d22ba4141f3531710e38df"
+dependencies = [
+ "convert_case",
+ "proc-macro2",
+ "quote",
+ "rustc_version 0.3.3",
+ "syn",
+]
+
 [[package]]
 name = "devise"
 version = "0.3.1"
@@ -820,9 +839,9 @@ checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"

 [[package]]
 name = "h2"
-version = "0.3.4"
+version = "0.3.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d7f3675cfef6a30c8031cf9e6493ebdc3bb3272a3fea3923c4210d1830e6a472"
+checksum = "825343c4eef0b63f541f8903f395dc5beb362a979b5799a84062527ef1e37726"
 dependencies = [
 "bytes",
 "fnv",
@@ -933,9 +952,9 @@ dependencies = [

 [[package]]
 name = "http-body"
-version = "0.4.3"
+version = "0.4.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "399c583b2979440c60be0821a6199eca73bc3c8dcd9d070d75ac726e2c6186e5"
+checksum = "60daa14be0e0786db0f03a9e57cb404c9d756eed2b6c62b9ea98ec5743ec75a9"
 dependencies = [
 "bytes",
 "http",
@@ -944,9 +963,9 @@ dependencies = [

 [[package]]
 name = "httparse"
-version = "1.5.1"
+version = "1.4.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acd94fdbe1d4ff688b67b04eee2e17bd50995534a61539e45adfefb45e5e5503"
+checksum = "f3a87b616e37e93c22fb19bcd386f02f3af5ea98a25670ad0fce773de23c5e68"

 [[package]]
 name = "httpdate"
@@ -965,9 +984,9 @@ dependencies = [

 [[package]]
 name = "hyper"
-version = "0.14.12"
+version = "0.14.11"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "13f67199e765030fa08fe0bd581af683f0d5bc04ea09c2b1102012c5fb90e7fd"
+checksum = "0b61cf2d1aebcf6e6352c97b81dc2244ca29194be1b276f5d8ad5c6330fffb11"
 dependencies = [
 "bytes",
 "futures-channel",
@@ -1025,7 +1044,7 @@ dependencies = [
 "gif",
 "jpeg-decoder",
 "num-iter",
- "num-rational 0.3.2",
+ "num-rational",
 "num-traits",
 "png",
 ]
@@ -1109,15 +1128,15 @@ dependencies = [

 [[package]]
 name = "itoa"
-version = "0.4.8"
+version = "0.4.7"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b71991ff56294aa922b450139ee08b3bfc70982c6b2c7562771375cf73542dd4"
+checksum = "dd25036021b0de88a0aff6b850051563c6516d0bf53f8638938edbb9de732736"

 [[package]]
 name = "jobserver"
-version = "0.1.24"
+version = "0.1.22"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
+checksum = "972f5ae5d1cb9c6ae417789196c803205313edde988685da5e3aae0827b9e7fd"
 dependencies = [
 "libc",
 ]
@@ -1130,9 +1149,9 @@ checksum = "229d53d58899083193af11e15917b5640cd40b29ff475a1fe4ef725deb02d0f2"

 [[package]]
 name = "js-sys"
-version = "0.3.53"
+version = "0.3.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e4bf49d50e2961077d9c99f4b7997d770a1114f087c3c2e0069b36c13fc2979d"
+checksum = "83bdfbace3a0e81a4253f73b49e960b053e396a11012cbd49b9b74d6a2b67062"
 dependencies = [
 "wasm-bindgen",
 ]
@@ -1168,9 +1187,9 @@ checksum = "e2abad23fbc42b3700f2f279844dc832adb2b2eb069b2df918f455c4e18cc646"

 [[package]]
 name = "libc"
-version = "0.2.101"
+version = "0.2.98"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3cb00336871be5ed2c8ed44b60ae9959dc5b9f08539422ed43f09e34ecaeba21"
+checksum = "320cfe77175da3a483efed4bc0adc1968ca050b098ce4f2f1c13a56626128790"

 [[package]]
 name = "libsqlite3-sys"
@@ -1263,15 +1282,15 @@ dependencies = [

 [[package]]
 name = "matches"
-version = "0.1.9"
+version = "0.1.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "a3e378b66a060d48947b590737b30a1be76706c8dd7b8ba0f2fe3989c68a853f"
+checksum = "7ffc5c5338469d4d3ea17d269fa8ea3512ad247247c30bd2df69e68309ed0a08"

 [[package]]
 name = "memchr"
-version = "2.4.1"
+version = "2.4.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "308cc39be01b73d0d18f82a0e7b2a3df85245f84af96fdddc5d202d27e47b86a"
+checksum = "b16bd47d9e329435e309c58469fe0791c2d0d1ba96ec0954152a5ae2b04387dc"

 [[package]]
 name = "memoffset"
@@ -1321,11 +1340,12 @@ dependencies = [

 [[package]]
 name = "multer"
-version = "2.0.1"
+version = "2.0.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "408327e2999b839cd1af003fc01b2019a6c10a1361769542203f6fedc5179680"
+checksum = "0fdd568fea4758b30d6423f013f7171e193c34aa97828d1bd9f924fb3af30a8c"
 dependencies = [
 "bytes",
+ "derive_more",
 "encoding_rs",
 "futures-util",
 "http",
@@ -1348,20 +1368,6 @@ dependencies = [
 "winapi",
 ]

-[[package]]
-name = "num"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "43db66d1170d347f9a065114077f7dccb00c1b9478c89384490a3425279a4606"
-dependencies = [
- "num-bigint 0.4.0",
- "num-complex",
- "num-integer",
- "num-iter",
- "num-rational 0.4.0",
- "num-traits",
-]
-
 [[package]]
 name = "num-bigint"
 version = "0.2.6"
@@ -1373,26 +1379,6 @@ dependencies = [
 "num-traits",
 ]

-[[package]]
-name = "num-bigint"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4e0d047c1062aa51e256408c560894e5251f08925980e53cf1aa5bd00eec6512"
-dependencies = [
- "autocfg",
- "num-integer",
- "num-traits",
-]
-
-[[package]]
-name = "num-complex"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "26873667bbbb7c5182d4a37c1add32cdf09f841af72da53318fdb81543c15085"
-dependencies = [
- "num-traits",
-]
-
 [[package]]
 name = "num-integer"
 version = "0.1.44"
@@ -1425,18 +1411,6 @@ dependencies = [
 "num-traits",
 ]

-[[package]]
-name = "num-rational"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d41702bd167c2df5520b384281bc111a4b5efcf7fbc4c9c222c815b07e0a6a6a"
-dependencies = [
- "autocfg",
- "num-bigint 0.4.0",
- "num-integer",
- "num-traits",
-]
-
 [[package]]
 name = "num-traits"
 version = "0.2.14"
@@ -1491,47 +1465,17 @@ dependencies = [
 "thiserror",
 ]

-[[package]]
-name = "opentelemetry"
-version = "0.16.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e1cf9b1c4e9a6c4de793c632496fa490bdc0e1eea73f0c91394f7b6990935d22"
-dependencies = [
- "async-trait",
- "crossbeam-channel",
- "futures",
- "js-sys",
- "lazy_static",
- "percent-encoding",
- "pin-project",
- "rand 0.8.4",
- "thiserror",
- "tokio",
- "tokio-stream",
-]
-
 [[package]]
 name = "opentelemetry-jaeger"
-version = "0.15.0"
+version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "db22f492873ea037bc267b35a0e8e4fb846340058cb7c864efe3d0bf23684593"
+checksum = "09a9fc8192722e7daa0c56e59e2336b797122fb8598383dcb11c8852733b435c"
 dependencies = [
 "async-trait",
 "lazy_static",
- "opentelemetry 0.16.0",
- "opentelemetry-semantic-conventions",
+ "opentelemetry",
 "thiserror",
 "thrift",
- "tokio",
-]
-
-[[package]]
-name = "opentelemetry-semantic-conventions"
-version = "0.8.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "ffeac823339e8b0f27b961f4385057bf9f97f2863bc745bd015fd6091f2270e9"
-dependencies = [
- "opentelemetry 0.16.0",
 ]

 [[package]]
@@ -1624,6 +1568,15 @@ version = "2.1.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d4fd5641d01c8f18a23da7b6fe29298ff4b55afcccdf78973b24cf3175fee32e"

+[[package]]
+name = "pest"
+version = "2.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "10f4872ae94d7b90ae48754df22fd42ad52ce740b8f370b03da4835417403e53"
+dependencies = [
+ "ucd-trie",
+]
+
 [[package]]
 name = "pin-project"
 version = "1.0.8"
@@ -1843,9 +1796,9 @@ dependencies = [

 [[package]]
 name = "redox_syscall"
-version = "0.2.10"
+version = "0.2.9"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8383f39639269cde97d255a32bdb68c047337295414940c68bdd30c2e13203ff"
+checksum = "5ab49abadf3f9e1c4bc499e8845e152ad87d2ad2d30371841171169e9d75feee"
 dependencies = [
 "bitflags",
 ]
@@ -2061,9 +2014,8 @@ dependencies = [

 [[package]]
 name = "ruma"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "668031e3108d6a2cfbe6eca271d8698f4593440e71a44afdadcf67ce3cb93c1f"
+version = "0.2.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "assign",
 "js_int",
@@ -2083,9 +2035,8 @@ dependencies = [

 [[package]]
 name = "ruma-api"
-version = "0.18.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "f5f1843792b6749ec1ece62595cf99ad30bf9589c96bb237515235e71da396ea"
+version = "0.17.1"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "bytes",
 "http",
@@ -2100,9 +2051,8 @@ dependencies = [

 [[package]]
 name = "ruma-api-macros"
-version = "0.18.3"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7b18abda5cca94178d08b622bca042e1cbb5eb7d4ebf3a2a81590a3bb3c57008"
+version = "0.17.1"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@@ -2112,9 +2062,8 @@ dependencies = [

 [[package]]
 name = "ruma-appservice-api"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "49369332a5f299e832e19661f92d49e08c345c3c6c4ab16e09cb31c5ff6da878"
+version = "0.3.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "ruma-api",
 "ruma-common",
@@ -2127,9 +2076,8 @@ dependencies = [

 [[package]]
 name = "ruma-client-api"
-version = "0.12.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "9568a222c12cf6220e751484ab78feec28071f85965113a5bb802936a2920ff0"
+version = "0.11.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "assign",
 "bytes",
@@ -2148,9 +2096,8 @@ dependencies = [

 [[package]]
 name = "ruma-common"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "41d5b7605f58dc0d9cf1848cc7f1af2bae4e4bcd1d2b7a87bbb9864c8a785b91"
+version = "0.5.4"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "indexmap",
 "js_int",
@@ -2164,9 +2111,8 @@ dependencies = [

 [[package]]
 name = "ruma-events"
-version = "0.24.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "87801e1207cfebdee02e7997ebf181a1c9837260b78c1b8ce96b896a2bcb3763"
+version = "0.23.2"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "indoc",
 "js_int",
@@ -2181,9 +2127,8 @@ dependencies = [

 [[package]]
 name = "ruma-events-macros"
-version = "0.24.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5da4498845347de88adf1b7da4578e2ca7355ad4ce47b0976f6594bacf958660"
+version = "0.23.2"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@@ -2193,9 +2138,8 @@ dependencies = [

 [[package]]
 name = "ruma-federation-api"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c61c9adbe1a29c301ae627604406d60102c89fc833b110cd35bbf29ae205ea6c"
+version = "0.2.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "js_int",
 "ruma-api",
@@ -2209,9 +2153,8 @@ dependencies = [

 [[package]]
 name = "ruma-identifiers"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cb417d091e8dd5a633e4e5998231a156049d7fcc221045cfdc0642eb72067732"
+version = "0.19.4"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "paste",
 "rand 0.8.4",
@@ -2224,9 +2167,8 @@ dependencies = [

 [[package]]
 name = "ruma-identifiers-macros"
-version = "0.20.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c708edad7f605638f26c951cbad7501fbf28ab01009e5ca65ea5a2db74a882b1"
+version = "0.19.4"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "quote",
 "ruma-identifiers-validation",
@@ -2235,15 +2177,13 @@ dependencies = [

 [[package]]
 name = "ruma-identifiers-validation"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "42285e7fb5d5f2d5268e45bb683e36d5c6fd9fc1e11a4559ba3c3521f3bbb2cb"
+version = "0.4.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"

 [[package]]
 name = "ruma-identity-service-api"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e76e66e24f2d5a31511fbf6c79e79f67a7a6a98ebf48d72381b7d5bb6c09f035"
+version = "0.2.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "js_int",
 "ruma-api",
@@ -2255,9 +2195,8 @@ dependencies = [

 [[package]]
 name = "ruma-push-gateway-api"
-version = "0.3.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5ef5b29da7065efc5b1e1a8f61add7543c9ab4ecce5ee0dd1c1c5ecec83fbeec"
+version = "0.2.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "js_int",
 "ruma-api",
@@ -2271,9 +2210,8 @@ dependencies = [

 [[package]]
 name = "ruma-serde"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8b2b22aae842e7ecda695e42b7b39d4558959d9d9a27acc2a16acf4f4f7f00c3"
+version = "0.4.1"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "bytes",
 "form_urlencoded",
@@ -2286,9 +2224,8 @@ dependencies = [

 [[package]]
 name = "ruma-serde-macros"
-version = "0.5.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "243e9bef188b08f94c79bc2f8fd1eb307a9e636b2b8e4571acf8c7be16381d28"
+version = "0.4.1"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@@ -2298,9 +2235,8 @@ dependencies = [

 [[package]]
 name = "ruma-signatures"
-version = "0.9.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4a4f64027165b59500162d10d435b1253898bf3ad4f5002cb0d56913fe7f76d7"
+version = "0.8.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "base64 0.13.0",
 "ed25519-dalek",
@@ -2316,9 +2252,8 @@ dependencies = [

 [[package]]
 name = "ruma-state-res"
-version = "0.4.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "796427aaa2d266238c5c1b1a6ca4640a4d282ec2cb2e844c69a8f3a262d3db15"
+version = "0.2.0"
+source = "git+https://github.com/timokoesters/ruma?rev=a2d93500e1dbc87e7032a3c74f3b2479a7f84e93#a2d93500e1dbc87e7032a3c74f3b2479a7f84e93"
 dependencies = [
 "itertools 0.10.1",
 "js_int",
@@ -2366,7 +2301,16 @@ version = "0.2.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "138e3e0acb6c9fb258b19b67cb8abd63c00679d2851805ea151465464fe9030a"
 dependencies = [
- "semver",
+ "semver 0.9.0",
+]
+
+[[package]]
+name = "rustc_version"
+version = "0.3.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f0dfe2087c51c460008730de8b57e6a320782fbfb312e1f4d520e6c6fae155ee"
+dependencies = [
+ "semver 0.11.0",
 ]

 [[package]]
@@ -2440,23 +2384,22 @@ dependencies = [

 [[package]]
 name = "security-framework"
-version = "2.4.0"
+version = "2.3.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "5b9bd29cdffb8875b04f71c51058f940cf4e390bbfd2ce669c4f22cd70b492a5"
+checksum = "23a2ac85147a3a11d77ecf1bc7166ec0b92febfa4461c37944e180f319ece467"
 dependencies = [
 "bitflags",
 "core-foundation",
 "core-foundation-sys",
 "libc",
- "num",
 "security-framework-sys",
 ]

 [[package]]
 name = "security-framework-sys"
-version = "2.4.0"
+version = "2.3.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "19133a286e494cc3311c165c4676ccb1fd47bed45b55f9d71fbd784ad4cea6f8"
+checksum = "7e4effb91b4b8b6fb7732e670b6cee160278ff8e6bf485c7805d9e319d76e284"
 dependencies = [
 "core-foundation-sys",
 "libc",
@@ -2468,7 +2411,16 @@ version = "0.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "1d7eb9ef2c18661902cc47e535f9bc51b78acd254da71d375c2f6720d9a40403"
 dependencies = [
- "semver-parser",
+ "semver-parser 0.7.0",
+]
+
+[[package]]
+name = "semver"
+version = "0.11.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f301af10236f6df4160f7c3f04eec6dbc70ace82d23326abad5edee88801c6b6"
+dependencies = [
+ "semver-parser 0.10.2",
 ]

 [[package]]
@@ -2478,19 +2430,28 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "388a1df253eca08550bef6c72392cfe7c30914bf41df5269b68cbd6ff8f570a3"

 [[package]]
-name = "serde"
-version = "1.0.129"
+name = "semver-parser"
+version = "0.10.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d1f72836d2aa753853178eda473a3b9d8e4eefdaf20523b919677e6de489f8f1"
+checksum = "00b0bef5b7f9e0df16536d3961cfb6e84331c065b4066afb39768d0e319411f7"
+dependencies = [
+ "pest",
+]
+
+[[package]]
+name = "serde"
+version = "1.0.126"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ec7505abeacaec74ae4778d9d9328fe5a5d04253220a85c4ee022239fc996d03"
 dependencies = [
 "serde_derive",
 ]

 [[package]]
 name = "serde_derive"
-version = "1.0.129"
+version = "1.0.126"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "e57ae87ad533d9a56427558b516d0adac283614e347abf85b0dc0cbbf0a249f3"
+checksum = "963a7dbc9895aeac7ac90e74f34a5d5261828f79df35cbed41e10189d3804d43"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -2522,12 +2483,12 @@ dependencies = [

 [[package]]
 name = "serde_yaml"
-version = "0.8.19"
+version = "0.8.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6375dbd828ed6964c3748e4ef6d18e7a175d408ffe184bca01698d0c73f915a9"
+checksum = "15654ed4ab61726bf918a39cb8d98a2e2995b002387807fa6ba58fdf7f59bb23"
 dependencies = [
 "dtoa",
- "indexmap",
+ "linked-hash-map",
 "serde",
 "yaml-rust",
 ]
@@ -2553,9 +2514,9 @@ dependencies = [

 [[package]]
 name = "sharded-slab"
-version = "0.1.3"
+version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "740223c51853f3145fe7c90360d2d4232f2b62e3449489c207eccde818979982"
+checksum = "79c719719ee05df97490f80a45acfc99e5a30ce98a1e4fb67aee422745ae14e3"
 dependencies = [
 "lazy_static",
 ]
@@ -2582,15 +2543,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "692ca13de57ce0613a363c8c2f1de925adebc81b04c923ac60c5488bb44abe4b"
 dependencies = [
 "chrono",
- "num-bigint 0.2.6",
+ "num-bigint",
 "num-traits",
 ]

 [[package]]
 name = "slab"
-version = "0.4.4"
+version = "0.4.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "c307a32c1c5c437f38c7fd45d753050587732ba8628319fbdf12a7e289ccc590"
+checksum = "f173ac3d1a7e3b28003f40de0b5ce7fe2710f9b9dc3fc38664cebee46b3b6527"

 [[package]]
 name = "sled"
@@ -2691,7 +2652,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d022496b16281348b52d0e30ae99e01a73d737b2f45d38fed4edf79f9325a1d5"
 dependencies = [
 "discard",
- "rustc_version",
+ "rustc_version 0.2.3",
 "stdweb-derive",
 "stdweb-internal-macros",
 "stdweb-internal-runtime",
@@ -2741,9 +2702,9 @@ checksum = "6bdef32e8150c2a081110b42772ffe7d7c9032b606bc226c8260fd97e0976601"

 [[package]]
 name = "syn"
-version = "1.0.75"
+version = "1.0.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b7f58f7e8eaa0009c5fec437aabf511bd9933e4b2d7407bd05273c01a8906ea7"
+checksum = "1873d832550d4588c3dbc20f01361ab00bfe741048f71e3fecf145a7cc18b29c"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -2910,9 +2871,9 @@ checksum = "cda74da7e1a664f795bb1f8a87ec406fb89a02522cf6e50620d016add6dbbf5c"

 [[package]]
 name = "tokio"
-version = "1.10.1"
+version = "1.9.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "92036be488bb6594459f2e03b60e42df6f937fe6ca5c5ffdcb539c6b84dc40f5"
+checksum = "4b7b349f11a7047e6d1276853e612d152f5e8a352c61917887cc2169e2366b4c"
 dependencies = [
 "autocfg",
 "bytes",
@@ -3026,9 +2987,9 @@ dependencies = [

 [[package]]
 name = "tracing-core"
-version = "0.1.19"
+version = "0.1.18"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "2ca517f43f0fb96e0c3072ed5c275fe5eece87e8cb52f4a77b69226d3b1c9df8"
+checksum = "a9ff14f98b1a4b289c6248a023c1c2fa1491062964e9fed67ab29c4e4da4a052"
 dependencies = [
 "lazy_static",
 ]
@@ -3061,7 +3022,7 @@ version = "0.14.0"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "c47440f2979c4cd3138922840eec122e3c0ba2148bc290f756bd7fd60fc97fff"
 dependencies = [
- "opentelemetry 0.15.0",
+ "opentelemetry",
 "tracing",
 "tracing-core",
 "tracing-log",
@@ -3080,9 +3041,9 @@ dependencies = [

 [[package]]
 name = "tracing-subscriber"
-version = "0.2.20"
+version = "0.2.19"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b9cbe87a2fa7e35900ce5de20220a582a9483a7063811defce79d7cbd59d4cfe"
+checksum = "ab69019741fca4d98be3c62d2b75254528b5432233fd8a4d2739fec20278de48"
 dependencies = [
 "ansi_term",
 "chrono",
@@ -3176,6 +3137,12 @@ dependencies = [
 "serde",
 ]

+[[package]]
+name = "ucd-trie"
+version = "0.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "56dee185309b50d1f11bfedef0fe6d036842e3fb77413abef29f8f8d1c5d4c1c"
+
 [[package]]
 name = "uncased"
 version = "0.9.6"
@@ -3194,9 +3161,12 @@ checksum = "eeba86d422ce181a719445e51872fa30f1f7413b62becb52e95ec91aa262d85c"

 [[package]]
 name = "unicode-bidi"
-version = "0.3.6"
+version = "0.3.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "246f4c42e67e7a4e3c6106ff716a5d067d4132a642840b242e357e468a2a0085"
+checksum = "eeb8be209bb1c96b7c177c7420d26e04eccacb0eeae6b980e35fcb74678107e0"
+dependencies = [
+ "matches",
+]

 [[package]]
 name = "unicode-normalization"
@@ -3279,9 +3249,9 @@ checksum = "fd6fbd9a79829dd1ad0cc20627bf1ed606756a7f77edff7b66b7064f9cb327c6"

 [[package]]
 name = "wasm-bindgen"
-version = "0.2.76"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "8ce9b1b516211d33767048e5d47fa2a381ed8b76fc48d2ce4aa39877f9f183e0"
+checksum = "d54ee1d4ed486f78874278e63e4069fc1ab9f6a18ca492076ffb90c5eb2997fd"
 dependencies = [
 "cfg-if 1.0.0",
 "serde",
@@ -3291,9 +3261,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-backend"
-version = "0.2.76"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "cfe8dc78e2326ba5f845f4b5bf548401604fa20b1dd1d365fb73b6c1d6364041"
+checksum = "3b33f6a0694ccfea53d94db8b2ed1c3a8a4c86dd936b13b9f0a15ec4a451b900"
 dependencies = [
 "bumpalo",
 "lazy_static",
@@ -3306,9 +3276,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-futures"
-version = "0.4.26"
+version = "0.4.24"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "95fded345a6559c2cfee778d562300c581f7d4ff3edb9b0d230d69800d213972"
+checksum = "5fba7978c679d53ce2d0ac80c8c175840feb849a161664365d1287b41f2e67f1"
 dependencies = [
 "cfg-if 1.0.0",
 "js-sys",
@@ -3318,9 +3288,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-macro"
-version = "0.2.76"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "44468aa53335841d9d6b6c023eaab07c0cd4bddbcfdee3e2bb1e8d2cb8069fef"
+checksum = "088169ca61430fe1e58b8096c24975251700e7b1f6fd91cc9d59b04fb9b18bd4"
 dependencies = [
 "quote",
 "wasm-bindgen-macro-support",
@@ -3328,9 +3298,9 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-macro-support"
-version = "0.2.76"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "0195807922713af1e67dc66132c7328206ed9766af3858164fb583eedc25fbad"
+checksum = "be2241542ff3d9f241f5e2cb6dd09b37efe786df8851c54957683a49f0987a97"
 dependencies = [
 "proc-macro2",
 "quote",
@@ -3341,15 +3311,15 @@ dependencies = [

 [[package]]
 name = "wasm-bindgen-shared"
-version = "0.2.76"
+version = "0.2.74"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "acdb075a845574a1fa5f09fd77e43f7747599301ea3417a9fbffdeedfc1f4a29"
+checksum = "d7cff876b8f18eed75a66cf49b65e7f967cb354a7aa16003fb55dbfd25b44b4f"

 [[package]]
 name = "web-sys"
-version = "0.3.53"
+version = "0.3.51"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "224b2f6b67919060055ef1a67807367c2066ed520c3862cc013d26cf893a783c"
+checksum = "e828417b379f3df7111d3a2a9e5753706cae29c41f7c4029ee9fd77f3e09e582"
 dependencies = [
 "js-sys",
 "wasm-bindgen",
@@ -18,9 +18,8 @@ edition = "2018"
 rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests

 # Used for matrix spec type definitions and helpers
-ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
-#ruma = { git = "https://github.com/ruma/ruma", rev = "f5ab038e22421ed338396ece977b6b2844772ced", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
-#ruma = { git = "https://github.com/timokoesters/ruma", rev = "2215049b60a1c3358f5a52215adf1e7bb88619a1", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+#ruma = { git = "https://github.com/ruma/ruma", rev = "eb19b0e08a901b87d11b3be0890ec788cc760492", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+ruma = { git = "https://github.com/timokoesters/ruma", rev = "a2d93500e1dbc87e7032a3c74f3b2479a7f84e93", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }

 # Used for long polling and federation sender, should be the same as rocket::tokio
@@ -67,11 +66,11 @@ regex = "1.5.4"
 jsonwebtoken = "7.2.0"
 # Performance measurements
 tracing = { version = "0.1.26", features = ["release_max_level_warn"] }
+opentelemetry = "0.15.0"
 tracing-subscriber = "0.2.19"
 tracing-opentelemetry = "0.14.0"
 tracing-flame = "0.1.0"
-opentelemetry = { version = "0.16.0", features = ["rt-tokio"] }
-opentelemetry-jaeger = { version = "0.15.0", features = ["rt-tokio"] }
+opentelemetry-jaeger = "0.14.0"
 pretty_env_logger = "0.4.0"
 lru-cache = "0.1.2"
 rusqlite = { version = "0.25.3", optional = true, features = ["bundled"] }
@@ -120,20 +119,6 @@ conf-files = [
 maintainer-scripts = "debian/"
 systemd-units = { unit-name = "matrix-conduit" }

-[profile.dev]
-lto = 'thin'
-incremental = true
-
-[profile.release]
-lto = 'thin'
-incremental = true
-
-codegen-units=32
-# If you want to make flamegraphs, enable debug info:
-# debug = true
-
-# For releases also try to max optimizations for dependencies:
-[profile.release.build-override]
-opt-level = 3
-[profile.release.package."*"]
-opt-level = 3
+# For flamegraphs:
+#[profile.release]
+#debug = true
@@ -6,7 +6,8 @@ If you run into any problems while setting up Conduit, write an email to `timo@k

 ## Installing Conduit

-You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:
+You may simply download the binary that fits your machine. Run `uname -m` to see
+what you need. Now copy the right url:

 | CPU Architecture     | GNU (Ubuntu, Debian, ArchLinux, ...)  | MUSL (Alpine, ... )     |
 | -------------------- | ------------------------------------- | ----------------------- |
@@ -14,13 +15,10 @@ You may simply download the binary that fits your machine. Run `uname -m` to see
 | armv7 (Raspberry Pi) | [Download][armv7-gnu]                 | -                       |
 | armv8 / aarch64      | [Download][armv8-gnu]                 | -                       |

-[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:release:cargo:x86_64-unknown-linux-gnu
-
-[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
-
-[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:release:cargo:armv7-unknown-linux-gnueabihf
-
-[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:release:cargo:aarch64-unknown-linux-gnu
+[x84_64-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-gnu?job=build:cargo:x86_64-unknown-linux-gnu
+[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:cargo:x86_64-unknown-linux-musl
+[armv7-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-gnueabihf?job=build:cargo:armv7-unknown-linux-gnueabihf
+[armv8-gnu]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-gnu?job=build:cargo:aarch64-unknown-linux-gnu

 ```bash
 $ sudo wget -O /usr/local/bin/matrix-conduit <url>
@@ -28,7 +26,6 @@ $ sudo chmod +x /usr/local/bin/matrix-conduit
 ```

 Alternatively, you may compile the binary yourself using
-
 ```bash
 $ cargo build --release
 ```
@@ -249,8 +249,6 @@ pub async fn register_route(

        let room_id = RoomId::new(db.globals.server_name());

-        db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
-
        let mutex_state = Arc::clone(
            db.globals
                .roomid_mutex_state
@@ -292,7 +290,6 @@ pub async fn register_route(
                    is_direct: None,
                    third_party_invite: None,
                    blurhash: None,
-                    reason: None,
                })
                .expect("event is valid, we just created it"),
                unsigned: None,
@@ -458,7 +455,6 @@ pub async fn register_route(
                    is_direct: None,
                    third_party_invite: None,
                    blurhash: None,
-                    reason: None,
                })
                .expect("event is valid, we just created it"),
                unsigned: None,
@@ -480,7 +476,6 @@ pub async fn register_route(
                    is_direct: None,
                    third_party_invite: None,
                    blurhash: None,
-                    reason: None,
                })
                .expect("event is valid, we just created it"),
                unsigned: None,
@@ -686,7 +681,6 @@ pub async fn deactivate_route(
            is_direct: None,
            third_party_invite: None,
            blurhash: None,
-            reason: None,
        };

        let mutex_state = Arc::clone(
@@ -737,7 +731,7 @@ pub async fn deactivate_route(
 pub async fn third_party_route(
    body: Ruma<get_contacts::Request>,
 ) -> ConduitResult<get_contacts::Response> {
-    let _sender_user = body.sender_user.as_ref().expect("user is authenticated");
+    let sender_user = body.sender_user.as_ref().expect("user is authenticated");

    Ok(get_contacts::Response::new(Vec::new()).into())
 }
@@ -44,7 +44,7 @@ pub async fn get_context_route(

    let events_before = db
        .rooms
-        .pdus_until(&sender_user, &body.room_id, base_token)?
+        .pdus_until(&sender_user, &body.room_id, base_token)
        .take(
            u32::try_from(body.limit).map_err(|_| {
                Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.")
@@ -66,7 +66,7 @@ pub async fn get_context_route(

    let events_after = db
        .rooms
-        .pdus_after(&sender_user, &body.room_id, base_token)?
+        .pdus_after(&sender_user, &body.room_id, base_token)
        .take(
            u32::try_from(body.limit).map_err(|_| {
                Error::BadRequest(ErrorKind::InvalidParam, "Limit value is invalid.")
@@ -262,7 +262,6 @@ pub async fn ban_user_route(
                is_direct: None,
                third_party_invite: None,
                blurhash: db.users.blurhash(&body.user_id)?,
-                reason: None,
            }),
            |event| {
                let mut event = serde_json::from_value::<Raw<member::MemberEventContent>>(
@@ -564,7 +563,6 @@ async fn join_room_by_id_helper(
                is_direct: None,
                third_party_invite: None,
                blurhash: db.users.blurhash(&sender_user)?,
-                reason: None,
            })
            .expect("event is valid, we just created it"),
        );
@@ -611,8 +609,6 @@ async fn join_room_by_id_helper(
            )
            .await?;

-        db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
-
        let pdu = PduEvent::from_id_val(&event_id, join_event.clone())
            .map_err(|_| Error::BadServerResponse("Invalid join event PDU."))?;

@@ -640,40 +636,23 @@ async fn join_room_by_id_helper(

            db.rooms.add_pdu_outlier(&event_id, &value)?;
            if let Some(state_key) = &pdu.state_key {
-                let shortstatekey =
-                    db.rooms
-                        .get_or_create_shortstatekey(&pdu.kind, state_key, &db.globals)?;
-                state.insert(shortstatekey, pdu.event_id.clone());
+                state.insert((pdu.kind.clone(), state_key.clone()), pdu.event_id.clone());
            }
        }

-        let incoming_shortstatekey = db.rooms.get_or_create_shortstatekey(
-            &pdu.kind,
-            pdu.state_key
-                .as_ref()
-                .expect("Pdu is a membership state event"),
-            &db.globals,
-        )?;
+        state.insert(
+            (
+                pdu.kind.clone(),
+                pdu.state_key.clone().expect("join event has state key"),
+            ),
+            pdu.event_id.clone(),
+        );

-        state.insert(incoming_shortstatekey, pdu.event_id.clone());
-
-        let create_shortstatekey = db
-            .rooms
-            .get_shortstatekey(&EventType::RoomCreate, "")?
-            .expect("Room exists");
-
-        if state.get(&create_shortstatekey).is_none() {
+        if state.get(&(EventType::RoomCreate, "".to_owned())).is_none() {
            return Err(Error::BadServerResponse("State contained no create event."));
        }

-        db.rooms.force_state(
-            room_id,
-            state
-                .into_iter()
-                .map(|(k, id)| db.rooms.compress_state_event(k, &id, &db.globals))
-                .collect::<Result<HashSet<_>>>()?,
-            &db,
-        )?;
+        db.rooms.force_state(room_id, state, &db)?;

        for result in futures::future::join_all(
            send_join_response
@@ -714,7 +693,6 @@ async fn join_room_by_id_helper(
            is_direct: None,
            third_party_invite: None,
            blurhash: db.users.blurhash(&sender_user)?,
-            reason: None,
        };

        db.rooms.build_and_append_pdu(
@@ -839,10 +817,7 @@ pub async fn invite_helper<'a>(
                    serde_json::from_value::<Raw<CreateEventContent>>(create_event.content.clone())
                        .expect("Raw::from_value always works.")
                        .deserialize()
-                        .map_err(|e| {
-                            warn!("Invalid create event: {}", e);
-                            Error::bad_database("Invalid create event in db.")
-                        })
+                        .map_err(|_| Error::bad_database("Invalid PowerLevels event in db."))
                })
                .transpose()?;

@@ -869,7 +844,6 @@ pub async fn invite_helper<'a>(
                membership: MembershipState::Invite,
                third_party_invite: None,
                blurhash: None,
-                reason: None,
            })
            .expect("member event is valid value");

@@ -930,8 +904,8 @@ pub async fn invite_helper<'a>(
                &room_version,
                &Arc::new(pdu.clone()),
                create_prev_event,
+                &auth_events,
                None, // TODO: third_party_invite
-                |k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone),
            )
            .map_err(|e| {
                error!("{:?}", e);
@@ -1064,7 +1038,6 @@ pub async fn invite_helper<'a>(
                is_direct: Some(is_direct),
                third_party_invite: None,
                blurhash: db.users.blurhash(&user_id)?,
-                reason: None,
            })
            .expect("event is valid, we just created it"),
            unsigned: None,
@@ -128,7 +128,7 @@ pub async fn get_message_events_route(
        get_message_events::Direction::Forward => {
            let events_after = db
                .rooms
-                .pdus_after(&sender_user, &body.room_id, from)?
+                .pdus_after(&sender_user, &body.room_id, from)
                .take(limit)
                .filter_map(|r| r.ok()) // Filter out buggy events
                .filter_map(|(pdu_id, pdu)| {
@@ -158,7 +158,7 @@ pub async fn get_message_events_route(
        get_message_events::Direction::Backward => {
            let events_before = db
                .rooms
-                .pdus_until(&sender_user, &body.room_id, from)?
+                .pdus_until(&sender_user, &body.room_id, from)
                .take(limit)
                .filter_map(|r| r.ok()) // Filter out buggy events
                .filter_map(|(pdu_id, pdu)| {
@@ -33,8 +33,6 @@ pub async fn create_room_route(

    let room_id = RoomId::new(db.globals.server_name());

-    db.rooms.get_or_create_shortroomid(&room_id, &db.globals)?;
-
    let mutex_state = Arc::clone(
        db.globals
            .roomid_mutex_state
@@ -45,24 +43,24 @@ pub async fn create_room_route(
    );
    let state_lock = mutex_state.lock().await;

-    let alias: Option<RoomAliasId> =
-        body.room_alias_name
-            .as_ref()
-            .map_or(Ok(None), |localpart| {
-                // TODO: Check for invalid characters and maximum length
-                let alias =
-                    RoomAliasId::try_from(format!("#{}:{}", localpart, db.globals.server_name()))
-                        .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?;
+    let alias = body
+        .room_alias_name
+        .as_ref()
+        .map_or(Ok(None), |localpart| {
+            // TODO: Check for invalid characters and maximum length
+            let alias =
+                RoomAliasId::try_from(format!("#{}:{}", localpart, db.globals.server_name()))
+                    .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Invalid alias."))?;

-                if db.rooms.id_from_alias(&alias)?.is_some() {
-                    Err(Error::BadRequest(
-                        ErrorKind::RoomInUse,
-                        "Room alias already exists.",
-                    ))
-                } else {
-                    Ok(Some(alias))
-                }
-            })?;
+            if db.rooms.id_from_alias(&alias)?.is_some() {
+                Err(Error::BadRequest(
+                    ErrorKind::RoomInUse,
+                    "Room alias already exists.",
+                ))
+            } else {
+                Ok(Some(alias))
+            }
+        })?;

    let mut content = ruma::events::room::create::CreateEventContent::new(sender_user.clone());
    content.federate = body.creation_content.federate;
@@ -107,7 +105,6 @@ pub async fn create_room_route(
                is_direct: Some(body.is_direct),
                third_party_invite: None,
                blurhash: db.users.blurhash(&sender_user)?,
-                reason: None,
            })
            .expect("event is valid, we just created it"),
            unsigned: None,
@@ -175,32 +172,9 @@ pub async fn create_room_route(
        &state_lock,
    )?;

-    // 4. Canonical room alias
-    if let Some(room_alias_id) = &alias {
-        db.rooms.build_and_append_pdu(
-            PduBuilder {
-                event_type: EventType::RoomCanonicalAlias,
-                content: serde_json::to_value(
-                    ruma::events::room::canonical_alias::CanonicalAliasEventContent {
-                        alias: Some(room_alias_id.clone()),
-                        alt_aliases: vec![],
-                    },
-                )
-                .expect("We checked that alias earlier, it must be fine"),
-                unsigned: None,
-                state_key: Some("".to_owned()),
-                redacts: None,
-            },
-            &sender_user,
-            &room_id,
-            &db,
-            &state_lock,
-        )?;
-    }
+    // 4. Events set by preset

-    // 5. Events set by preset
-
-    // 5.1 Join Rules
+    // 4.1 Join Rules
    db.rooms.build_and_append_pdu(
        PduBuilder {
            event_type: EventType::RoomJoinRules,
@@ -225,7 +199,7 @@ pub async fn create_room_route(
        &state_lock,
    )?;

-    // 5.2 History Visibility
+    // 4.2 History Visibility
    db.rooms.build_and_append_pdu(
        PduBuilder {
            event_type: EventType::RoomHistoryVisibility,
@@ -243,7 +217,7 @@ pub async fn create_room_route(
        &state_lock,
    )?;

-    // 5.3 Guest Access
+    // 4.3 Guest Access
    db.rooms.build_and_append_pdu(
        PduBuilder {
            event_type: EventType::RoomGuestAccess,
@@ -269,7 +243,7 @@ pub async fn create_room_route(
        &state_lock,
    )?;

-    // 6. Events listed in initial_state
+    // 5. Events listed in initial_state
    for event in &body.initial_state {
        let pdu_builder = PduBuilder::from(event.deserialize().map_err(|e| {
            warn!("Invalid initial state event: {:?}", e);
@@ -285,7 +259,7 @@ pub async fn create_room_route(
            .build_and_append_pdu(pdu_builder, &sender_user, &room_id, &db, &state_lock)?;
    }

-    // 7. Events implied by name and topic
+    // 6. Events implied by name and topic
    if let Some(name) = &body.name {
        db.rooms.build_and_append_pdu(
            PduBuilder {
@@ -322,7 +296,7 @@ pub async fn create_room_route(
        )?;
    }

-    // 8. Events implied by invite (and TODO: invite_3pid)
+    // 7. Events implied by invite (and TODO: invite_3pid)
    drop(state_lock);
    for user_id in &body.invite {
        let _ = invite_helper(sender_user, user_id, &room_id, &db, body.is_direct).await;
@@ -423,8 +397,6 @@ pub async fn upgrade_room_route(

    // Create a replacement room
    let replacement_room = RoomId::new(db.globals.server_name());
-    db.rooms
-        .get_or_create_shortroomid(&replacement_room, &db.globals)?;

    let mutex_state = Arc::clone(
        db.globals
@@ -520,7 +492,6 @@ pub async fn upgrade_room_route(
                is_direct: None,
                third_party_invite: None,
                blurhash: db.users.blurhash(&sender_user)?,
-                reason: None,
            })
            .expect("event is valid, we just created it"),
            unsigned: None,
@@ -3,10 +3,7 @@ use crate::{database::DatabaseGuard, utils, ConduitResult, Error, Ruma};
 use ruma::{
    api::client::{
        error::ErrorKind,
-        r0::{
-            session::{get_login_types, login, logout, logout_all},
-            uiaa::IncomingUserIdentifier,
-        },
+        r0::session::{get_login_types, login, logout, logout_all},
    },
    UserId,
 };
@@ -63,7 +60,7 @@ pub async fn login_route(
            identifier,
            password,
        } => {
-            let username = if let IncomingUserIdentifier::MatrixId(matrix_id) = identifier {
+            let username = if let login::IncomingUserIdentifier::MatrixId(matrix_id) = identifier {
                matrix_id
            } else {
                return Err(Error::BadRequest(ErrorKind::Forbidden, "Bad login type."));
@@ -12,7 +12,7 @@ use std::{
    time::Duration,
 };
 use tokio::sync::watch::Sender;
-use tracing::error;
+use tracing::{error, warn};

 #[cfg(feature = "conduit_bin")]
 use rocket::{get, tokio};
@@ -205,7 +205,7 @@ async fn sync_helper(

        let mut non_timeline_pdus = db
            .rooms
-            .pdus_until(&sender_user, &room_id, u64::MAX)?
+            .pdus_until(&sender_user, &room_id, u64::MAX)
            .filter_map(|r| {
                // Filter out buggy events
                if r.is_err() {
@@ -246,13 +246,31 @@ async fn sync_helper(
            .current_shortstatehash(&room_id)?
            .expect("All rooms have state");

+        let first_pdu_before_since = db
+            .rooms
+            .pdus_until(&sender_user, &room_id, since)
+            .next()
+            .transpose()?;
+
        let pdus_after_since = db
            .rooms
-            .pdus_after(&sender_user, &room_id, since)?
+            .pdus_after(&sender_user, &room_id, since)
            .next()
            .is_some();

-        let since_shortstatehash = db.rooms.get_token_shortstatehash(&room_id, since)?;
+        let since_shortstatehash = first_pdu_before_since
+            .as_ref()
+            .map(|pdu| {
+                db.rooms
+                    .pdu_shortstatehash(&pdu.1.event_id)
+                    .transpose()
+                    .ok_or_else(|| {
+                        warn!("PDU without state: {}", pdu.1.event_id);
+                        Error::bad_database("Found PDU without state")
+                    })
+            })
+            .transpose()?
+            .transpose()?;

        // Calculates joined_member_count, invited_member_count and heroes
        let calculate_counts = || {
@@ -268,7 +286,7 @@ async fn sync_helper(

                for hero in db
                    .rooms
-                    .all_pdus(&sender_user, &room_id)?
+                    .all_pdus(&sender_user, &room_id)
                    .filter_map(|pdu| pdu.ok()) // Ignore all broken pdus
                    .filter(|(_, pdu)| pdu.kind == EventType::RoomMember)
                    .map(|(_, pdu)| {
@@ -310,11 +328,11 @@ async fn sync_helper(
                }
            }

-            Ok::<_, Error>((
+            (
                Some(joined_member_count),
                Some(invited_member_count),
                heroes,
-            ))
+            )
        };

        let (
@@ -325,12 +343,12 @@ async fn sync_helper(
            state_events,
        ) = if since_shortstatehash.is_none() {
            // Probably since = 0, we will do an initial sync
-            let (joined_member_count, invited_member_count, heroes) = calculate_counts()?;
+            let (joined_member_count, invited_member_count, heroes) = calculate_counts();

            let current_state_ids = db.rooms.state_full_ids(current_shortstatehash)?;
            let state_events = current_state_ids
                .iter()
-                .map(|(_, id)| db.rooms.get_pdu(id))
+                .map(|id| db.rooms.get_pdu(id))
                .filter_map(|r| r.ok().flatten())
                .collect::<Vec<_>>();

@@ -341,7 +359,7 @@ async fn sync_helper(
                true,
                state_events,
            )
-        } else if !pdus_after_since && since_shortstatehash == Some(current_shortstatehash) {
+        } else if !pdus_after_since || since_shortstatehash == Some(current_shortstatehash) {
            // No state changes
            (Vec::new(), None, None, false, Vec::new())
        } else {
@@ -375,14 +393,18 @@ async fn sync_helper(
            let state_events = if joined_since_last_sync {
                current_state_ids
                    .iter()
-                    .map(|(_, id)| db.rooms.get_pdu(id))
+                    .map(|id| db.rooms.get_pdu(id))
                    .filter_map(|r| r.ok().flatten())
                    .collect::<Vec<_>>()
            } else {
                current_state_ids
-                    .iter()
-                    .filter(|(key, id)| since_state_ids.get(key) != Some(id))
-                    .map(|(_, id)| db.rooms.get_pdu(id))
+                    .difference(&since_state_ids)
+                    .filter(|id| {
+                        !timeline_pdus
+                            .iter()
+                            .any(|(_, timeline_pdu)| timeline_pdu.event_id == **id)
+                    })
+                    .map(|id| db.rooms.get_pdu(id))
                    .filter_map(|r| r.ok().flatten())
                    .collect()
            };
@@ -488,7 +510,7 @@ async fn sync_helper(
            }

            let (joined_member_count, invited_member_count, heroes) = if send_member_count {
-                calculate_counts()?
+                calculate_counts()
            } else {
                (None, None, Vec::new())
            };
@@ -562,10 +584,6 @@ async fn sync_helper(
            );
        }

-        // Save the state after this sync so we can send the correct state diff next sync
-        db.rooms
-            .associate_token_shortstatehash(&room_id, next_batch, current_shortstatehash)?;
-
        let joined_room = sync_events::JoinedRoom {
            account_data: sync_events::RoomAccountData {
                events: db
@@ -56,7 +56,6 @@ pub async fn send_event_to_device_route(
                        },
                    ))
                    .expect("DirectToDevice EDU can be serialized"),
-                    db.globals.next_count()?,
                )?;

                continue;
@@ -1,34 +1,3 @@
-use std::hash::Hash;
-use std::{
-    collections::{BTreeMap, HashMap, HashSet},
-    convert::{TryFrom, TryInto},
-    fs::{self, remove_dir_all},
-    io::Write,
-    mem::size_of,
-    ops::Deref,
-    path::Path,
-    sync::{Arc, Mutex, RwLock},
-};
-
-use directories::ProjectDirs;
-use lru_cache::LruCache;
-use rocket::{
-    futures::{channel::mpsc, stream::FuturesUnordered, StreamExt},
-    outcome::{try_outcome, IntoOutcome},
-    request::{FromRequest, Request},
-    Shutdown, State,
-};
-use ruma::{DeviceId, EventId, RoomId, ServerName, UserId};
-use serde::{de::IgnoredAny, Deserialize};
-use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore};
-use tracing::{debug, error, warn};
-
-use abstraction::DatabaseEngine;
-
-use crate::{utils, Error, Result};
-
-use self::proxy::ProxyConfig;
-
 pub mod abstraction;

 pub mod account_data;
@@ -45,6 +14,32 @@ pub mod transaction_ids;
 pub mod uiaa;
 pub mod users;

+use crate::{utils, Error, Result};
+use abstraction::DatabaseEngine;
+use directories::ProjectDirs;
+use lru_cache::LruCache;
+use rocket::{
+    futures::{channel::mpsc, stream::FuturesUnordered, StreamExt},
+    outcome::{try_outcome, IntoOutcome},
+    request::{FromRequest, Request},
+    Shutdown, State,
+};
+use ruma::{DeviceId, RoomId, ServerName, UserId};
+use serde::{de::IgnoredAny, Deserialize};
+use std::{
+    collections::{BTreeMap, HashMap},
+    convert::TryFrom,
+    fs::{self, remove_dir_all},
+    io::Write,
+    ops::Deref,
+    path::Path,
+    sync::{Arc, Mutex, RwLock},
+};
+use tokio::sync::{OwnedRwLockReadGuard, RwLock as TokioRwLock, Semaphore};
+use tracing::{debug, error, warn};
+
+use self::proxy::ProxyConfig;
+
 #[derive(Clone, Debug, Deserialize)]
 pub struct Config {
    server_name: Box<ServerName>,
@@ -112,7 +107,7 @@ fn default_db_cache_capacity_mb() -> f64 {
 }

 fn default_sqlite_wal_clean_second_interval() -> u32 {
-    1 * 60 // every minute
+    15 * 60 // every 15 minutes
 }

 fn default_max_request_size() -> u32 {
@@ -136,17 +131,6 @@ pub type Engine = abstraction::sqlite::Engine;
 #[cfg(feature = "heed")]
 pub type Engine = abstraction::heed::Engine;

-// for each key: (memory_usage in bytes, items in cache, capacity)
-pub struct CacheUsageStatistics {
-    pdu_cache: (usize, usize, usize),
-    auth_chain_cache: (usize, usize, usize),
-    shorteventid_cache: (usize, usize, usize),
-    eventidshort_cache: (usize, usize, usize),
-    statekeyshort_cache: (usize, usize, usize),
-    shortstatekey_cache: (usize, usize, usize),
-    stateinfo_cache: (usize, usize, usize),
-}
-
 pub struct Database {
    _db: Arc<Engine>,
    pub globals: globals::Globals,
@@ -277,29 +261,18 @@ impl Database {
                userroomid_highlightcount: builder.open_tree("userroomid_highlightcount")?,

                statekey_shortstatekey: builder.open_tree("statekey_shortstatekey")?,
-                shortstatekey_statekey: builder.open_tree("shortstatekey_statekey")?,
-
-                shorteventid_authchain: builder.open_tree("shorteventid_authchain")?,
-
-                roomid_shortroomid: builder.open_tree("roomid_shortroomid")?,
-
-                shortstatehash_statediff: builder.open_tree("shortstatehash_statediff")?,
+                stateid_shorteventid: builder.open_tree("stateid_shorteventid")?,
                eventid_shorteventid: builder.open_tree("eventid_shorteventid")?,
                shorteventid_eventid: builder.open_tree("shorteventid_eventid")?,
                shorteventid_shortstatehash: builder.open_tree("shorteventid_shortstatehash")?,
                roomid_shortstatehash: builder.open_tree("roomid_shortstatehash")?,
-                roomsynctoken_shortstatehash: builder.open_tree("roomsynctoken_shortstatehash")?,
                statehash_shortstatehash: builder.open_tree("statehash_shortstatehash")?,

                eventid_outlierpdu: builder.open_tree("eventid_outlierpdu")?,
                referencedevents: builder.open_tree("referencedevents")?,
                pdu_cache: Mutex::new(LruCache::new(100_000)),
-                auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
+                auth_chain_cache: Mutex::new(LruCache::new(100_000)),
                shorteventid_cache: Mutex::new(LruCache::new(1_000_000)),
-                eventidshort_cache: Mutex::new(LruCache::new(1_000_000)),
-                shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)),
-                statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)),
-                stateinfo_cache: Mutex::new(LruCache::new(1000)),
            },
            account_data: account_data::AccountData {
                roomuserdataid_accountdata: builder.open_tree("roomuserdataid_accountdata")?,
@@ -452,6 +425,7 @@ impl Database {
            }

            if db.globals.database_version()? < 6 {
+                // TODO update to 6
                // Set room member count
                for (roomid, _) in db.rooms.roomid_shortstatehash.iter() {
                    let room_id =
@@ -464,279 +438,6 @@ impl Database {

                println!("Migration: 5 -> 6 finished");
            }
-
-            if db.globals.database_version()? < 7 {
-                // Upgrade state store
-                let mut last_roomstates: HashMap<RoomId, u64> = HashMap::new();
-                let mut current_sstatehash: Option<u64> = None;
-                let mut current_room = None;
-                let mut current_state = HashSet::new();
-                let mut counter = 0;
-
-                let mut handle_state =
-                    |current_sstatehash: u64,
-                     current_room: &RoomId,
-                     current_state: HashSet<_>,
-                     last_roomstates: &mut HashMap<_, _>| {
-                        counter += 1;
-                        println!("counter: {}", counter);
-                        let last_roomsstatehash = last_roomstates.get(current_room);
-
-                        let states_parents = last_roomsstatehash.map_or_else(
-                            || Ok(Vec::new()),
-                            |&last_roomsstatehash| {
-                                db.rooms.load_shortstatehash_info(dbg!(last_roomsstatehash))
-                            },
-                        )?;
-
-                        let (statediffnew, statediffremoved) =
-                            if let Some(parent_stateinfo) = states_parents.last() {
-                                let statediffnew = current_state
-                                    .difference(&parent_stateinfo.1)
-                                    .cloned()
-                                    .collect::<HashSet<_>>();
-
-                                let statediffremoved = parent_stateinfo
-                                    .1
-                                    .difference(&current_state)
-                                    .cloned()
-                                    .collect::<HashSet<_>>();
-
-                                (statediffnew, statediffremoved)
-                            } else {
-                                (current_state, HashSet::new())
-                            };
-
-                        db.rooms.save_state_from_diff(
-                            dbg!(current_sstatehash),
-                            statediffnew,
-                            statediffremoved,
-                            2, // every state change is 2 event changes on average
-                            states_parents,
-                        )?;
-
-                        /*
-                        let mut tmp = db.rooms.load_shortstatehash_info(&current_sstatehash, &db)?;
-                        let state = tmp.pop().unwrap();
-                        println!(
-                            "{}\t{}{:?}: {:?} + {:?} - {:?}",
-                            current_room,
-                            "  ".repeat(tmp.len()),
-                            utils::u64_from_bytes(&current_sstatehash).unwrap(),
-                            tmp.last().map(|b| utils::u64_from_bytes(&b.0).unwrap()),
-                            state
-                                .2
-                                .iter()
-                                .map(|b| utils::u64_from_bytes(&b[size_of::<u64>()..]).unwrap())
-                                .collect::<Vec<_>>(),
-                            state
-                                .3
-                                .iter()
-                                .map(|b| utils::u64_from_bytes(&b[size_of::<u64>()..]).unwrap())
-                                .collect::<Vec<_>>()
-                        );
-                        */
-
-                        Ok::<_, Error>(())
-                    };
-
-                for (k, seventid) in db._db.open_tree("stateid_shorteventid")?.iter() {
-                    let sstatehash = utils::u64_from_bytes(&k[0..size_of::<u64>()])
-                        .expect("number of bytes is correct");
-                    let sstatekey = k[size_of::<u64>()..].to_vec();
-                    if Some(sstatehash) != current_sstatehash {
-                        if let Some(current_sstatehash) = current_sstatehash {
-                            handle_state(
-                                current_sstatehash,
-                                current_room.as_ref().unwrap(),
-                                current_state,
-                                &mut last_roomstates,
-                            )?;
-                            last_roomstates
-                                .insert(current_room.clone().unwrap(), current_sstatehash);
-                        }
-                        current_state = HashSet::new();
-                        current_sstatehash = Some(sstatehash);
-
-                        let event_id = db
-                            .rooms
-                            .shorteventid_eventid
-                            .get(&seventid)
-                            .unwrap()
-                            .unwrap();
-                        let event_id =
-                            EventId::try_from(utils::string_from_bytes(&event_id).unwrap())
-                                .unwrap();
-                        let pdu = db.rooms.get_pdu(&event_id).unwrap().unwrap();
-
-                        if Some(&pdu.room_id) != current_room.as_ref() {
-                            current_room = Some(pdu.room_id.clone());
-                        }
-                    }
-
-                    let mut val = sstatekey;
-                    val.extend_from_slice(&seventid);
-                    current_state.insert(val.try_into().expect("size is correct"));
-                }
-
-                if let Some(current_sstatehash) = current_sstatehash {
-                    handle_state(
-                        current_sstatehash,
-                        current_room.as_ref().unwrap(),
-                        current_state,
-                        &mut last_roomstates,
-                    )?;
-                }
-
-                db.globals.bump_database_version(7)?;
-
-                println!("Migration: 6 -> 7 finished");
-            }
-
-            if db.globals.database_version()? < 8 {
-                // Generate short room ids for all rooms
-                for (room_id, _) in db.rooms.roomid_shortstatehash.iter() {
-                    let shortroomid = db.globals.next_count()?.to_be_bytes();
-                    db.rooms.roomid_shortroomid.insert(&room_id, &shortroomid)?;
-                    println!("Migration: 8");
-                }
-                // Update pduids db layout
-                let mut batch = db.rooms.pduid_pdu.iter().filter_map(|(key, v)| {
-                    if !key.starts_with(b"!") {
-                        return None;
-                    }
-                    let mut parts = key.splitn(2, |&b| b == 0xff);
-                    let room_id = parts.next().unwrap();
-                    let count = parts.next().unwrap();
-
-                    let short_room_id = db
-                        .rooms
-                        .roomid_shortroomid
-                        .get(&room_id)
-                        .unwrap()
-                        .expect("shortroomid should exist");
-
-                    let mut new_key = short_room_id;
-                    new_key.extend_from_slice(count);
-
-                    Some((new_key, v))
-                });
-
-                db.rooms.pduid_pdu.insert_batch(&mut batch)?;
-
-                let mut batch2 = db.rooms.eventid_pduid.iter().filter_map(|(k, value)| {
-                    if !value.starts_with(b"!") {
-                        return None;
-                    }
-                    let mut parts = value.splitn(2, |&b| b == 0xff);
-                    let room_id = parts.next().unwrap();
-                    let count = parts.next().unwrap();
-
-                    let short_room_id = db
-                        .rooms
-                        .roomid_shortroomid
-                        .get(&room_id)
-                        .unwrap()
-                        .expect("shortroomid should exist");
-
-                    let mut new_value = short_room_id;
-                    new_value.extend_from_slice(count);
-
-                    Some((k, new_value))
-                });
-
-                db.rooms.eventid_pduid.insert_batch(&mut batch2)?;
-
-                db.globals.bump_database_version(8)?;
-
-                println!("Migration: 7 -> 8 finished");
-            }
-
-            if db.globals.database_version()? < 9 {
-                // Update tokenids db layout
-                let batch = db
-                    .rooms
-                    .tokenids
-                    .iter()
-                    .filter_map(|(key, _)| {
-                        if !key.starts_with(b"!") {
-                            return None;
-                        }
-                        let mut parts = key.splitn(4, |&b| b == 0xff);
-                        let room_id = parts.next().unwrap();
-                        let word = parts.next().unwrap();
-                        let _pdu_id_room = parts.next().unwrap();
-                        let pdu_id_count = parts.next().unwrap();
-
-                        let short_room_id = db
-                            .rooms
-                            .roomid_shortroomid
-                            .get(&room_id)
-                            .unwrap()
-                            .expect("shortroomid should exist");
-                        let mut new_key = short_room_id;
-                        new_key.extend_from_slice(word);
-                        new_key.push(0xff);
-                        new_key.extend_from_slice(pdu_id_count);
-                        println!("old {:?}", key);
-                        println!("new {:?}", new_key);
-                        Some((new_key, Vec::new()))
-                    })
-                    .collect::<Vec<_>>();
-
-                let mut iter = batch.into_iter().peekable();
-
-                while iter.peek().is_some() {
-                    db.rooms
-                        .tokenids
-                        .insert_batch(&mut iter.by_ref().take(1000))?;
-                    println!("smaller batch done");
-                }
-
-                println!("Deleting starts");
-
-                let batch2 = db
-                    .rooms
-                    .tokenids
-                    .iter()
-                    .filter_map(|(key, _)| {
-                        if key.starts_with(b"!") {
-                            println!("del {:?}", key);
-                            Some(key)
-                        } else {
-                            None
-                        }
-                    })
-                    .collect::<Vec<_>>();
-
-                for key in batch2 {
-                    println!("del");
-                    db.rooms.tokenids.remove(&key)?;
-                }
-
-                db.globals.bump_database_version(9)?;
-
-                println!("Migration: 8 -> 9 finished");
-            }
-
-            if db.globals.database_version()? < 10 {
-                // Add other direction for shortstatekeys
-                for (statekey, shortstatekey) in db.rooms.statekey_shortstatekey.iter() {
-                    db.rooms
-                        .shortstatekey_statekey
-                        .insert(&shortstatekey, &statekey)?;
-                }
-
-                // Force E2EE device list updates so we can send them over federation
-                for user_id in db.users.iter().filter_map(|r| r.ok()) {
-                    db.users
-                        .mark_device_key_update(&user_id, &db.rooms, &db.globals)?;
-                }
-
-                db.globals.bump_database_version(10)?;
-
-                println!("Migration: 9 -> 10 finished");
-            }
        }

        let guard = db.read().await;
@@ -932,66 +633,6 @@ impl Database {
            }
        });
    }
-
-    /// Measures memory usage in bytes and how full the caches are in percent for all caches in the Database struct.
-    pub fn get_cache_usage(&mut self) -> Result<CacheUsageStatistics> {
-        fn memory_usage_of_locked_cache<K: Eq + Hash, V>(
-            cache: &mut Mutex<LruCache<K, V>>,
-        ) -> usize {
-            let raw_cache = cache.lock().unwrap();
-            let mut cache_items_size_sum: usize = 0;
-            for cache_item in raw_cache.iter() {
-                cache_items_size_sum += std::mem::size_of_val(&cache_item);
-            }
-            cache_items_size_sum += std::mem::size_of_val(&cache);
-            cache_items_size_sum
-        }
-
-        fn items_in_locked_cache<K: Eq + Hash, V>(cache: &mut Mutex<LruCache<K, V>>) -> usize {
-            cache.lock().unwrap().len()
-        }
-        fn capacity_of_locked_cache<K: Eq + Hash, V>(cache: &mut Mutex<LruCache<K, V>>) -> usize {
-            cache.lock().unwrap().capacity()
-        }
-
-        return Ok(CacheUsageStatistics {
-            pdu_cache: (
-                memory_usage_of_locked_cache(&mut self.rooms.pdu_cache),
-                items_in_locked_cache(&mut self.rooms.pdu_cache),
-                capacity_of_locked_cache(&mut self.rooms.pdu_cache),
-            ),
-            auth_chain_cache: (
-                memory_usage_of_locked_cache(&mut self.rooms.auth_chain_cache),
-                items_in_locked_cache(&mut self.rooms.auth_chain_cache),
-                capacity_of_locked_cache(&mut self.rooms.auth_chain_cache),
-            ),
-            shorteventid_cache: (
-                memory_usage_of_locked_cache(&mut self.rooms.shorteventid_cache),
-                items_in_locked_cache(&mut self.rooms.shorteventid_cache),
-                capacity_of_locked_cache(&mut self.rooms.shorteventid_cache),
-            ),
-            eventidshort_cache: (
-                memory_usage_of_locked_cache(&mut self.rooms.eventidshort_cache),
-                items_in_locked_cache(&mut self.rooms.eventidshort_cache),
-                capacity_of_locked_cache(&mut self.rooms.eventidshort_cache),
-            ),
-            statekeyshort_cache: (
-                memory_usage_of_locked_cache(&mut self.rooms.statekeyshort_cache),
-                items_in_locked_cache(&mut self.rooms.statekeyshort_cache),
-                capacity_of_locked_cache(&mut self.rooms.statekeyshort_cache),
-            ),
-            shortstatekey_cache: (
-                memory_usage_of_locked_cache(&mut self.rooms.shortstatekey_cache),
-                items_in_locked_cache(&mut self.rooms.shortstatekey_cache),
-                capacity_of_locked_cache(&mut self.rooms.shortstatekey_cache),
-            ),
-            stateinfo_cache: (
-                memory_usage_of_locked_cache(&mut self.rooms.stateinfo_cache),
-                items_in_locked_cache(&mut self.rooms.stateinfo_cache),
-                capacity_of_locked_cache(&mut self.rooms.stateinfo_cache),
-            ),
-        });
-    }
 }

 pub struct DatabaseGuard(OwnedRwLockReadGuard<Database>);
@@ -35,7 +35,6 @@ pub trait Tree: Send + Sync {
    ) -> Box<dyn Iterator<Item = (Vec<u8>, Vec<u8>)> + 'a>;

    fn increment(&self, key: &[u8]) -> Result<Vec<u8>>;
-    fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()>;

    fn scan_prefix<'a>(
        &'a self,
@@ -9,13 +9,13 @@ use std::{
    path::{Path, PathBuf},
    pin::Pin,
    sync::Arc,
+    time::{Duration, Instant},
 };
 use tokio::sync::oneshot::Sender;
 use tracing::debug;

 thread_local! {
    static READ_CONNECTION: RefCell<Option<&'static Connection>> = RefCell::new(None);
-    static READ_CONNECTION_ITERATOR: RefCell<Option<&'static Connection>> = RefCell::new(None);
 }

 struct PreparedStatementIterator<'a> {
@@ -49,11 +49,11 @@ impl Engine {
    fn prepare_conn(path: &Path, cache_size_kb: u32) -> Result<Connection> {
        let conn = Connection::open(&path)?;

-        conn.pragma_update(Some(Main), "page_size", &2048)?;
+        conn.pragma_update(Some(Main), "page_size", &32768)?;
        conn.pragma_update(Some(Main), "journal_mode", &"WAL")?;
        conn.pragma_update(Some(Main), "synchronous", &"NORMAL")?;
        conn.pragma_update(Some(Main), "cache_size", &(-i64::from(cache_size_kb)))?;
-        conn.pragma_update(Some(Main), "wal_autocheckpoint", &2000)?;
+        conn.pragma_update(Some(Main), "wal_autocheckpoint", &0)?;

        Ok(conn)
    }
@@ -77,25 +77,9 @@ impl Engine {
        })
    }

-    fn read_lock_iterator(&self) -> &'static Connection {
-        READ_CONNECTION_ITERATOR.with(|cell| {
-            let connection = &mut cell.borrow_mut();
-
-            if (*connection).is_none() {
-                let c = Box::leak(Box::new(
-                    Self::prepare_conn(&self.path, self.cache_size_per_thread).unwrap(),
-                ));
-                **connection = Some(c);
-            }
-
-            connection.unwrap()
-        })
-    }
-
    pub fn flush_wal(self: &Arc<Self>) -> Result<()> {
-        // We use autocheckpoints
-        //self.write_lock()
-        //.pragma_update(Some(Main), "wal_checkpoint", &"TRUNCATE")?;
+        self.write_lock()
+            .pragma_update(Some(Main), "wal_checkpoint", &"TRUNCATE")?;
        Ok(())
    }
 }
@@ -167,34 +151,6 @@ impl SqliteTable {
        )?;
        Ok(())
    }
-
-    pub fn iter_with_guard<'a>(
-        &'a self,
-        guard: &'a Connection,
-    ) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
-        let statement = Box::leak(Box::new(
-            guard
-                .prepare(&format!(
-                    "SELECT key, value FROM {} ORDER BY key ASC",
-                    &self.name
-                ))
-                .unwrap(),
-        ));
-
-        let statement_ref = NonAliasingBox(statement);
-
-        let iterator = Box::new(
-            statement
-                .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
-                .unwrap()
-                .map(|r| r.unwrap()),
-        );
-
-        Box::new(PreparedStatementIterator {
-            iterator,
-            statement_ref,
-        })
-    }
 }

 impl Tree for SqliteTable {
@@ -249,24 +205,6 @@ impl Tree for SqliteTable {
        Ok(())
    }

-    #[tracing::instrument(skip(self, iter))]
-    fn increment_batch<'a>(&self, iter: &mut dyn Iterator<Item = Vec<u8>>) -> Result<()> {
-        let guard = self.engine.write_lock();
-
-        guard.execute("BEGIN", [])?;
-        for key in iter {
-            let old = self.get_with_guard(&guard, &key)?;
-            let new = crate::utils::increment(old.as_deref())
-                .expect("utils::increment always returns Some");
-            self.insert_with_guard(&guard, &key, &new)?;
-        }
-        guard.execute("COMMIT", [])?;
-
-        drop(guard);
-
-        Ok(())
-    }
-
    #[tracing::instrument(skip(self, key))]
    fn remove(&self, key: &[u8]) -> Result<()> {
        let guard = self.engine.write_lock();
@@ -281,9 +219,30 @@ impl Tree for SqliteTable {

    #[tracing::instrument(skip(self))]
    fn iter<'a>(&'a self) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
-        let guard = self.engine.read_lock_iterator();
+        let guard = self.engine.read_lock();

-        self.iter_with_guard(&guard)
+        let statement = Box::leak(Box::new(
+            guard
+                .prepare(&format!(
+                    "SELECT key, value FROM {} ORDER BY key ASC",
+                    &self.name
+                ))
+                .unwrap(),
+        ));
+
+        let statement_ref = NonAliasingBox(statement);
+
+        let iterator = Box::new(
+            statement
+                .query_map([], |row| Ok((row.get_unwrap(0), row.get_unwrap(1))))
+                .unwrap()
+                .map(|r| r.unwrap()),
+        );
+
+        Box::new(PreparedStatementIterator {
+            iterator,
+            statement_ref,
+        })
    }

    #[tracing::instrument(skip(self, from, backwards))]
@@ -292,7 +251,7 @@ impl Tree for SqliteTable {
        from: &[u8],
        backwards: bool,
    ) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
-        let guard = self.engine.read_lock_iterator();
+        let guard = self.engine.read_lock();
        let from = from.to_vec(); // TODO change interface?

        if backwards {
@@ -3,21 +3,19 @@ use std::{
    sync::Arc,
 };

+use crate::{pdu::PduBuilder, Database};
 use rocket::futures::{channel::mpsc, stream::StreamExt};
 use ruma::{
    events::{room::message, EventType},
    UserId,
 };
-use tokio::sync::{MutexGuard, RwLock, RwLockWriteGuard};
+use tokio::sync::{MutexGuard, RwLock, RwLockReadGuard};
 use tracing::warn;

-use crate::{pdu::PduBuilder, Database};
-
 pub enum AdminCommand {
    RegisterAppservice(serde_yaml::Value),
    ListAppservices,
    SendMessage(message::MessageEventContent),
-    ShowCacheUsage,
 }

 #[derive(Clone)]
@@ -61,7 +59,7 @@ impl Admin {
            drop(guard);

            let send_message = |message: message::MessageEventContent,
-                                guard: RwLockWriteGuard<'_, Database>,
+                                guard: RwLockReadGuard<'_, Database>,
                                mutex_lock: &MutexGuard<'_, ()>| {
                guard
                    .rooms
@@ -85,7 +83,7 @@ impl Admin {
            loop {
                tokio::select! {
                    Some(event) = receiver.next() => {
-                        let mut guard = db.write().await;
+                        let guard = db.read().await;
                        let mutex_state = Arc::clone(
                            guard.globals
                                .roomid_mutex_state
@@ -94,7 +92,6 @@ impl Admin {
                                .entry(conduit_room.clone())
                                .or_default(),
                        );
-
                        let state_lock = mutex_state.lock().await;

                        match event {
@@ -117,37 +114,6 @@ impl Admin {
                            AdminCommand::SendMessage(message) => {
                                send_message(message, guard, &state_lock);
                            }
-                            AdminCommand::ShowCacheUsage => {
-
-                                fn format_cache_statistics_triple(name: String, triple: (usize, usize, usize)) -> String {
-                                    let (memory_usage, item_count, capacity) = triple;
-                                    format!(
-                                        "{0} is using {1} MB ({2} bytes) of RAM at {3:.2}% utilization.",
-                                        name,
-                                        memory_usage / 100_00,
-                                        memory_usage ,
-                                        ((item_count as f32 / capacity as f32) * 100.0)
-                                    )
-                                }
-
-                                if let Ok(cache_usage_statistics) = guard.get_cache_usage() {
-
-                                    let mut statistics_lines =  Vec::with_capacity(7);
-                                    statistics_lines.push(format_cache_statistics_triple("pdu_cache".to_string(), cache_usage_statistics.pdu_cache));
-                                    statistics_lines.push(format_cache_statistics_triple("auth_chain_cache".to_string(), cache_usage_statistics.auth_chain_cache));
-                                    statistics_lines.push(format_cache_statistics_triple("shorteventid_cache".to_string(), cache_usage_statistics.shorteventid_cache));
-                                    statistics_lines.push(format_cache_statistics_triple("eventidshort_cache".to_string(), cache_usage_statistics.eventidshort_cache));
-                                    statistics_lines.push(format_cache_statistics_triple("statekeyshort_cache".to_string(), cache_usage_statistics.statekeyshort_cache));
-                                    statistics_lines.push(format_cache_statistics_triple("shortstatekey_cache".to_string(), cache_usage_statistics.shortstatekey_cache));
-                                    statistics_lines.push(format_cache_statistics_triple("stateinfo_cache".to_string(), cache_usage_statistics.stateinfo_cache));
-
-                                    send_message(message::MessageEventContent::text_plain(statistics_lines.join("\n")), guard, &state_lock);
-                                } else {
-                                    let result_text = "Could not calculate database cache size";
-                                    send_message(message::MessageEventContent::text_plain(result_text), guard, &state_lock);
-                                }
-
-                            }
                        }

                        drop(state_lock);
@@ -1,5 +1,5 @@
 use std::{
-    collections::{BTreeMap, HashMap, HashSet},
+    collections::{BTreeMap, HashMap},
    convert::{TryFrom, TryInto},
    fmt::Debug,
    sync::Arc,
@@ -20,17 +20,14 @@ use ruma::{
        appservice,
        federation::{
            self,
-            transactions::edu::{
-                DeviceListUpdateContent, Edu, ReceiptContent, ReceiptData, ReceiptMap,
-            },
+            transactions::edu::{Edu, ReceiptContent, ReceiptData, ReceiptMap},
        },
        OutgoingRequest,
    },
-    device_id,
    events::{push_rules, AnySyncEphemeralRoomEvent, EventType},
    push,
    receipt::ReceiptType,
-    uint, MilliSecondsSinceUnixEpoch, ServerName, UInt, UserId,
+    MilliSecondsSinceUnixEpoch, ServerName, UInt, UserId,
 };
 use tokio::{
    select,
@@ -84,8 +81,8 @@ pub enum SendingEventType {
 pub struct Sending {
    /// The state for a given state hash.
    pub(super) servername_educount: Arc<dyn Tree>, // EduCount: Count of last EDU sync
-    pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / Id (for edus), Data = EDU content
-    pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / Id (for edus), Data = EDU content
+    pub(super) servernameevent_data: Arc<dyn Tree>, // ServernamEvent = (+ / $)SenderKey / ServerName / UserId + PduId / * (for edus), Data = EDU content
+    pub(super) servercurrentevent_data: Arc<dyn Tree>, // ServerCurrentEvents = (+ / $)ServerName / UserId + PduId / * (for edus), Data = EDU content
    pub(super) maximum_requests: Arc<Semaphore>,
    pub sender: mpsc::UnboundedSender<(Vec<u8>, Vec<u8>)>,
 }
@@ -320,19 +317,8 @@ impl Sending {
            })?;
        let mut events = Vec::new();
        let mut max_edu_count = since;
-        let mut device_list_changes = HashSet::new();
-
        'outer: for room_id in db.rooms.server_rooms(server) {
            let room_id = room_id?;
-            // Look for device list updates in this room
-            device_list_changes.extend(
-                db.users
-                    .keys_changed(&room_id.to_string(), since, None)
-                    .filter_map(|r| r.ok())
-                    .filter(|user_id| user_id.server_name() == db.globals.server_name()),
-            );
-
-            // Look for read receipts in this room
            for r in db.rooms.edus.readreceipts_since(&room_id, since) {
                let (user_id, count, read_receipt) = r?;

@@ -392,22 +378,6 @@ impl Sending {
            }
        }

-        for user_id in device_list_changes {
-            // Empty prev id forces synapse to resync: https://github.com/matrix-org/synapse/blob/98aec1cc9da2bd6b8e34ffb282c85abf9b8b42ca/synapse/handlers/device.py#L767
-            // Because synapse resyncs, we can just insert dummy data
-            let edu = Edu::DeviceListUpdate(DeviceListUpdateContent {
-                user_id,
-                device_id: device_id!("dummy"),
-                device_display_name: "Dummy".to_owned(),
-                stream_id: uint!(1),
-                prev_id: Vec::new(),
-                deleted: None,
-                keys: None,
-            });
-
-            events.push(serde_json::to_vec(&edu).expect("json can be serialized"));
-        }
-
        Ok((events, max_edu_count))
    }

@@ -435,15 +405,10 @@ impl Sending {
    }

    #[tracing::instrument(skip(self, server, serialized))]
-    pub fn send_reliable_edu(
-        &self,
-        server: &ServerName,
-        serialized: Vec<u8>,
-        id: u64,
-    ) -> Result<()> {
+    pub fn send_reliable_edu(&self, server: &ServerName, serialized: Vec<u8>) -> Result<()> {
        let mut key = server.as_bytes().to_vec();
        key.push(0xff);
-        key.extend_from_slice(&id.to_be_bytes());
+        key.push(b'*');
        self.servernameevent_data.insert(&key, &serialized)?;
        self.sender.unbounded_send((key, serialized)).unwrap();

@@ -719,10 +684,10 @@ impl Sending {
                OutgoingKind::Appservice(Box::<ServerName>::try_from(server).map_err(|_| {
                    Error::bad_database("Invalid server string in server_currenttransaction")
                })?),
-                if value.is_empty() {
-                    SendingEventType::Pdu(event.to_vec())
-                } else {
+                if event.starts_with(b"*") {
                    SendingEventType::Edu(value)
+                } else {
+                    SendingEventType::Pdu(event.to_vec())
                },
            )
        } else if key.starts_with(b"$") {
@@ -737,10 +702,10 @@ impl Sending {
                .ok_or_else(|| Error::bad_database("Invalid bytes in servercurrentpdus."))?;
            (
                OutgoingKind::Push(user.to_vec(), pushkey.to_vec()),
-                if value.is_empty() {
-                    SendingEventType::Pdu(event.to_vec())
-                } else {
+                if event.starts_with(b"*") {
                    SendingEventType::Edu(value)
+                } else {
+                    SendingEventType::Pdu(event.to_vec())
                },
            )
        } else {
@@ -758,10 +723,10 @@ impl Sending {
                OutgoingKind::Normal(Box::<ServerName>::try_from(server).map_err(|_| {
                    Error::bad_database("Invalid server string in server_currenttransaction")
                })?),
-                if value.is_empty() {
-                    SendingEventType::Pdu(event.to_vec())
+                if event.starts_with(b"*") {
+                    SendingEventType::Edu(event[1..].to_vec())
                } else {
-                    SendingEventType::Edu(value)
+                    SendingEventType::Pdu(event.to_vec())
                },
            )
        })
@@ -4,14 +4,11 @@ use crate::{client_server::SESSION_ID_LENGTH, utils, Error, Result};
 use ruma::{
    api::client::{
        error::ErrorKind,
-        r0::uiaa::{
-            IncomingAuthData, IncomingPassword, IncomingUserIdentifier::MatrixId, UiaaInfo,
-        },
+        r0::uiaa::{IncomingAuthData, UiaaInfo},
    },
    signatures::CanonicalJsonValue,
    DeviceId, UserId,
 };
-use tracing::error;

 use super::abstraction::Tree;

@@ -52,91 +49,126 @@ impl Uiaa {
        users: &super::users::Users,
        globals: &super::globals::Globals,
    ) -> Result<(bool, UiaaInfo)> {
-        let mut uiaainfo = auth
-            .session()
-            .map(|session| self.get_uiaa_session(&user_id, &device_id, session))
-            .unwrap_or_else(|| Ok(uiaainfo.clone()))?;
+        if let IncomingAuthData::DirectRequest {
+            kind,
+            session,
+            auth_parameters,
+        } = &auth
+        {
+            let mut uiaainfo = session
+                .as_ref()
+                .map(|session| self.get_uiaa_session(&user_id, &device_id, session))
+                .unwrap_or_else(|| Ok(uiaainfo.clone()))?;

-        if uiaainfo.session.is_none() {
-            uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
-        }
+            if uiaainfo.session.is_none() {
+                uiaainfo.session = Some(utils::random_string(SESSION_ID_LENGTH));
+            }

-        match auth {
            // Find out what the user completed
-            IncomingAuthData::Password(IncomingPassword {
-                identifier,
-                password,
-                ..
-            }) => {
-                let username = match identifier {
-                    MatrixId(username) => username,
-                    _ => {
+            match &**kind {
+                "m.login.password" => {
+                    let identifier = auth_parameters.get("identifier").ok_or(Error::BadRequest(
+                        ErrorKind::MissingParam,
+                        "m.login.password needs identifier.",
+                    ))?;
+
+                    let identifier_type = identifier.get("type").ok_or(Error::BadRequest(
+                        ErrorKind::MissingParam,
+                        "Identifier needs a type.",
+                    ))?;
+
+                    if identifier_type != "m.id.user" {
                        return Err(Error::BadRequest(
                            ErrorKind::Unrecognized,
                            "Identifier type not recognized.",
-                        ))
+                        ));
                    }
-                };

-                let user_id =
-                    UserId::parse_with_server_name(username.clone(), globals.server_name())
+                    let username = identifier
+                        .get("user")
+                        .ok_or(Error::BadRequest(
+                            ErrorKind::MissingParam,
+                            "Identifier needs user field.",
+                        ))?
+                        .as_str()
+                        .ok_or(Error::BadRequest(
+                            ErrorKind::BadJson,
+                            "User is not a string.",
+                        ))?;
+
+                    let user_id = UserId::parse_with_server_name(username, globals.server_name())
                        .map_err(|_| {
-                            Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid.")
-                        })?;
+                        Error::BadRequest(ErrorKind::InvalidParam, "User ID is invalid.")
+                    })?;

-                // Check if password is correct
-                if let Some(hash) = users.password_hash(&user_id)? {
-                    let hash_matches =
-                        argon2::verify_encoded(&hash, password.as_bytes()).unwrap_or(false);
+                    let password = auth_parameters
+                        .get("password")
+                        .ok_or(Error::BadRequest(
+                            ErrorKind::MissingParam,
+                            "Password is missing.",
+                        ))?
+                        .as_str()
+                        .ok_or(Error::BadRequest(
+                            ErrorKind::BadJson,
+                            "Password is not a string.",
+                        ))?;

-                    if !hash_matches {
-                        uiaainfo.auth_error = Some(ruma::api::client::error::ErrorBody {
-                            kind: ErrorKind::Forbidden,
-                            message: "Invalid username or password.".to_owned(),
-                        });
-                        return Ok((false, uiaainfo));
+                    // Check if password is correct
+                    if let Some(hash) = users.password_hash(&user_id)? {
+                        let hash_matches =
+                            argon2::verify_encoded(&hash, password.as_bytes()).unwrap_or(false);
+
+                        if !hash_matches {
+                            uiaainfo.auth_error = Some(ruma::api::client::error::ErrorBody {
+                                kind: ErrorKind::Forbidden,
+                                message: "Invalid username or password.".to_owned(),
+                            });
+                            return Ok((false, uiaainfo));
+                        }
+                    }
+
+                    // Password was correct! Let's add it to `completed`
+                    uiaainfo.completed.push("m.login.password".to_owned());
+                }
+                "m.login.dummy" => {
+                    uiaainfo.completed.push("m.login.dummy".to_owned());
+                }
+                k => panic!("type not supported: {}", k),
+            }
+
+            // Check if a flow now succeeds
+            let mut completed = false;
+            'flows: for flow in &mut uiaainfo.flows {
+                for stage in &flow.stages {
+                    if !uiaainfo.completed.contains(stage) {
+                        continue 'flows;
                    }
                }
-
-                // Password was correct! Let's add it to `completed`
-                uiaainfo.completed.push("m.login.password".to_owned());
+                // We didn't break, so this flow succeeded!
+                completed = true;
            }
-            IncomingAuthData::Dummy(_) => {
-                uiaainfo.completed.push("m.login.dummy".to_owned());
-            }
-            k => error!("type not supported: {:?}", k),
-        }

-        // Check if a flow now succeeds
-        let mut completed = false;
-        'flows: for flow in &mut uiaainfo.flows {
-            for stage in &flow.stages {
-                if !uiaainfo.completed.contains(stage) {
-                    continue 'flows;
-                }
+            if !completed {
+                self.update_uiaa_session(
+                    user_id,
+                    device_id,
+                    uiaainfo.session.as_ref().expect("session is always set"),
+                    Some(&uiaainfo),
+                )?;
+                return Ok((false, uiaainfo));
            }
-            // We didn't break, so this flow succeeded!
-            completed = true;
-        }

-        if !completed {
+            // UIAA was successful! Remove this session and return true
            self.update_uiaa_session(
                user_id,
                device_id,
                uiaainfo.session.as_ref().expect("session is always set"),
-                Some(&uiaainfo),
+                None,
            )?;
-            return Ok((false, uiaainfo));
+            Ok((true, uiaainfo))
+        } else {
+            panic!("FallbackAcknowledgement is not supported yet");
        }
-
-        // UIAA was successful! Remove this session and return true
-        self.update_uiaa_session(
-            user_id,
-            device_id,
-            uiaainfo.session.as_ref().expect("session is always set"),
-            None,
-        )?;
-        Ok((true, uiaainfo))
    }

    fn set_uiaa_request(
@@ -673,7 +673,7 @@ impl Users {
    }

    #[tracing::instrument(skip(self, user_id, rooms, globals))]
-    pub fn mark_device_key_update(
+    fn mark_device_key_update(
        &self,
        user_id: &UserId,
        rooms: &super::rooms::Rooms,
@@ -17,7 +17,7 @@ use std::sync::Arc;
 use database::Config;
 pub use database::Database;
 pub use error::{Error, Result};
-use opentelemetry::trace::{FutureExt, Tracer};
+use opentelemetry::trace::Tracer;
 pub use pdu::PduEvent;
 pub use rocket::State;
 use ruma::api::client::error::ErrorKind;
@@ -160,8 +160,7 @@ fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<
                server_server::get_room_state_route,
                server_server::get_room_state_ids_route,
                server_server::create_join_event_template_route,
-                server_server::create_join_event_v1_route,
-                server_server::create_join_event_v2_route,
+                server_server::create_join_event_route,
                server_server::create_invite_route,
                server_server::get_devices_route,
                server_server::get_room_information_route,
@@ -221,17 +220,14 @@ async fn main() {
    };

    if config.allow_jaeger {
-        opentelemetry::global::set_text_map_propagator(opentelemetry_jaeger::Propagator::new());
        let tracer = opentelemetry_jaeger::new_pipeline()
-            .install_batch(opentelemetry::runtime::Tokio)
+            .with_service_name("conduit")
+            .install_simple()
            .unwrap();

        let span = tracer.start("conduit");
-        start.with_current_context().await;
+        start.await;
        drop(span);
-
-        println!("exporting");
-        opentelemetry::global::shutdown_tracer_provider();
    } else {
        std::env::set_var("RUST_LOG", &config.log);

@@ -12,7 +12,7 @@ use ruma::{
 use serde::{Deserialize, Serialize};
 use serde_json::json;
 use std::{cmp::Ordering, collections::BTreeMap, convert::TryFrom};
-use tracing::warn;
+use tracing::error;

 #[derive(Clone, Deserialize, Serialize, Debug)]
 pub struct PduEvent {
@@ -322,7 +322,7 @@ pub(crate) fn gen_event_id_canonical_json(
    pdu: &Raw<ruma::events::pdu::Pdu>,
 ) -> crate::Result<(EventId, CanonicalJsonObject)> {
    let value = serde_json::from_str(pdu.json().get()).map_err(|e| {
-        warn!("Error parsing incoming event {:?}: {:?}", pdu, e);
+        error!("{:?}: {:?}", pdu, e);
        Error::BadServerResponse("Invalid PDU in server response")
    })?;
Author	SHA1	Message	Date
Timo Kösters	3e79b15bbf	fix: long prev event fetch times for huge rooms	2021-08-12 17:55:16 +02:00
Timo Kösters	b0a2877f20	less warnings	2021-08-11 21:17:01 +02:00
Timo Kösters	e46f2199c9	improvement: cache for short event ids	2021-08-11 21:14:22 +02:00
Timo Kösters	03d78b25f2	improvement: use u64s in auth chain cache	2021-08-11 19:15:38 +02:00
Timo Kösters	67749364aa	improvement: smaller cache, better prev event fetching	2021-08-11 10:24:16 +02:00