Merge branch 'adjust-ci-for-next-and-master-development' into 'master'

CI: Adjust CI for master and next branch development model (master) See merge request famedly/conduit!187
chore(CI): Adjust CI for master and next branch development model
2021-09-02 18:25:57 +00:00 · 2021-09-02 17:17:51 +02:00 · 2021-09-02 11:10:23 +00:00 · 2021-09-02 13:05:19 +02:00
6 changed files with 203 additions and 107 deletions
@@ -18,7 +18,8 @@ variables:
  stage: "build"
  needs: []
  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
    - if: '$CI_COMMIT_TAG'
  interruptible: true
  image: "rust:latest"
@@ -42,6 +43,8 @@ variables:
  script:
    - time cargo build --target $TARGET --release
    - 'mv "target/$TARGET/release/conduit" "conduit-$TARGET"'
+  artifacts:
+    expire_in: never


 build:release:cargo:x86_64-unknown-linux-gnu:
@@ -113,6 +116,8 @@ build:release:cargo:x86_64-unknown-linux-musl:
  script:
    - "time cargo build --target $TARGET"
    - 'mv "target/$TARGET/debug/conduit" "conduit-debug-$TARGET"'
+  artifacts:
+    expire_in: 4 weeks

 build:debug:cargo:x86_64-unknown-linux-gnu:
  extends: ".cargo-debug-shared-settings"
@@ -152,7 +157,8 @@ build:debug:cargo:x86_64-unknown-linux-musl:
  stage: "build"
  needs: [ ]
  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
    - if: '$CI_COMMIT_TAG'
  interruptible: true
  image: "rust:latest"
@@ -187,11 +193,11 @@ build:cargo-deb:x86_64-unknown-linux-gnu:
    expose_as: "Debian Package x86_64"


-
 # --------------------------------------------------------------------- #
 #  Create and publish docker image                                      #
 # --------------------------------------------------------------------- #

+# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
 .docker-shared-settings:
  stage: "build docker image"
  needs: []
@@ -208,8 +214,30 @@ build:cargo-deb:x86_64-unknown-linux-gnu:
    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'


-# Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
-build:docker:main:
+build:docker:next:
+  extends: .docker-shared-settings
+  needs:
+    - "build:release:cargo:x86_64-unknown-linux-musl"
+  script:
+    - >
+      /kaniko/executor
+      $KANIKO_CACHE_ARGS
+      --context $CI_PROJECT_DIR
+      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
+      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
+      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
+      --destination "$CI_REGISTRY_IMAGE/conduit:next"
+      --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine"
+      --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "next"'
+
+
+build:docker:master:
  extends: .docker-shared-settings
  needs:
    - "build:release:cargo:x86_64-unknown-linux-musl"
@@ -223,13 +251,31 @@ build:docker:main:
      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
      --destination "$CI_REGISTRY_IMAGE/conduit:latest"
-      --destination "$CI_REGISTRY_IMAGE/conduit:alpine"
-      --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
+      --destination "$CI_REGISTRY_IMAGE/conduit:latest-alpine"
      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
-      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:alpine"
-      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest-alpine"
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master"'
+
+
+build:docker:tags:
+  extends: .docker-shared-settings
+  needs:
+    - "build:release:cargo:x86_64-unknown-linux-musl"
+  script:
+    - >
+      /kaniko/executor
+      $KANIKO_CACHE_ARGS
+      --context $CI_PROJECT_DIR
+      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
+      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
+      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
+      --destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG"
+      --destination "$CI_REGISTRY_IMAGE/conduit:$CI_COMMIT_TAG-alpine"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG"
+      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:$CI_COMMIT_TAG-alpine"
  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
    - if: '$CI_COMMIT_TAG'


@@ -331,7 +377,8 @@ publish:package:
    - "build:release:cargo:x86_64-unknown-linux-musl"
    - "build:cargo-deb:x86_64-unknown-linux-gnu"
  rules:
-    - if: '$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH'
+    - if: '$CI_COMMIT_BRANCH == "master"'
+    - if: '$CI_COMMIT_BRANCH == "next"'
    - if: '$CI_COMMIT_TAG'
  image: curlimages/curl:latest
  tags: ["docker"]
@@ -325,9 +325,9 @@ checksum = "ea221b5284a47e40033bf9b66f35f984ec0ea2931eb03505246cd27a963f981b"

 [[package]]
 name = "cpufeatures"
-version = "0.1.5"
+version = "0.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "66c99696f6c9dd7f35d486b9d04d7e6e202aa3e8c40d553f2fdf5e7e0c6a71ef"
+checksum = "95059428f66df56b63431fdb4e1947ed2190586af5c5a8a8b71122bdf5a7f469"
 dependencies = [
 "libc",
 ]
@@ -2000,7 +2000,8 @@ dependencies = [
 [[package]]
 name = "ruma"
 version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "668031e3108d6a2cfbe6eca271d8698f4593440e71a44afdadcf67ce3cb93c1f"
 dependencies = [
 "assign",
 "js_int",
@@ -2021,7 +2022,8 @@ dependencies = [
 [[package]]
 name = "ruma-api"
 version = "0.18.3"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f5f1843792b6749ec1ece62595cf99ad30bf9589c96bb237515235e71da396ea"
 dependencies = [
 "bytes",
 "http",
@@ -2037,7 +2039,8 @@ dependencies = [
 [[package]]
 name = "ruma-api-macros"
 version = "0.18.3"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7b18abda5cca94178d08b622bca042e1cbb5eb7d4ebf3a2a81590a3bb3c57008"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@@ -2048,7 +2051,8 @@ dependencies = [
 [[package]]
 name = "ruma-appservice-api"
 version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "49369332a5f299e832e19661f92d49e08c345c3c6c4ab16e09cb31c5ff6da878"
 dependencies = [
 "ruma-api",
 "ruma-common",
@@ -2062,7 +2066,8 @@ dependencies = [
 [[package]]
 name = "ruma-client-api"
 version = "0.12.2"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9568a222c12cf6220e751484ab78feec28071f85965113a5bb802936a2920ff0"
 dependencies = [
 "assign",
 "bytes",
@@ -2082,7 +2087,8 @@ dependencies = [
 [[package]]
 name = "ruma-common"
 version = "0.6.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "41d5b7605f58dc0d9cf1848cc7f1af2bae4e4bcd1d2b7a87bbb9864c8a785b91"
 dependencies = [
 "indexmap",
 "js_int",
@@ -2097,7 +2103,8 @@ dependencies = [
 [[package]]
 name = "ruma-events"
 version = "0.24.5"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87801e1207cfebdee02e7997ebf181a1c9837260b78c1b8ce96b896a2bcb3763"
 dependencies = [
 "indoc",
 "js_int",
@@ -2113,7 +2120,8 @@ dependencies = [
 [[package]]
 name = "ruma-events-macros"
 version = "0.24.5"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5da4498845347de88adf1b7da4578e2ca7355ad4ce47b0976f6594bacf958660"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@@ -2124,7 +2132,8 @@ dependencies = [
 [[package]]
 name = "ruma-federation-api"
 version = "0.3.1"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fa3d1db1a064ab26484df6ef5d96c384fc053022004f34d96c3b4939e13dc204"
 dependencies = [
 "js_int",
 "ruma-api",
@@ -2139,7 +2148,8 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers"
 version = "0.20.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "cb417d091e8dd5a633e4e5998231a156049d7fcc221045cfdc0642eb72067732"
 dependencies = [
 "paste",
 "rand 0.8.4",
@@ -2153,7 +2163,8 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-macros"
 version = "0.20.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c708edad7f605638f26c951cbad7501fbf28ab01009e5ca65ea5a2db74a882b1"
 dependencies = [
 "quote",
 "ruma-identifiers-validation",
@@ -2163,15 +2174,14 @@ dependencies = [
 [[package]]
 name = "ruma-identifiers-validation"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
-dependencies = [
- "thiserror",
-]
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "42285e7fb5d5f2d5268e45bb683e36d5c6fd9fc1e11a4559ba3c3521f3bbb2cb"

 [[package]]
 name = "ruma-identity-service-api"
 version = "0.3.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e76e66e24f2d5a31511fbf6c79e79f67a7a6a98ebf48d72381b7d5bb6c09f035"
 dependencies = [
 "js_int",
 "ruma-api",
@@ -2184,7 +2194,8 @@ dependencies = [
 [[package]]
 name = "ruma-push-gateway-api"
 version = "0.3.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5ef5b29da7065efc5b1e1a8f61add7543c9ab4ecce5ee0dd1c1c5ecec83fbeec"
 dependencies = [
 "js_int",
 "ruma-api",
@@ -2199,7 +2210,8 @@ dependencies = [
 [[package]]
 name = "ruma-serde"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b2b22aae842e7ecda695e42b7b39d4558959d9d9a27acc2a16acf4f4f7f00c3"
 dependencies = [
 "bytes",
 "form_urlencoded",
@@ -2213,7 +2225,8 @@ dependencies = [
 [[package]]
 name = "ruma-serde-macros"
 version = "0.5.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "243e9bef188b08f94c79bc2f8fd1eb307a9e636b2b8e4571acf8c7be16381d28"
 dependencies = [
 "proc-macro-crate",
 "proc-macro2",
@@ -2224,7 +2237,8 @@ dependencies = [
 [[package]]
 name = "ruma-signatures"
 version = "0.9.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "4a4f64027165b59500162d10d435b1253898bf3ad4f5002cb0d56913fe7f76d7"
 dependencies = [
 "base64 0.13.0",
 "ed25519-dalek",
@@ -2240,8 +2254,9 @@ dependencies = [

 [[package]]
 name = "ruma-state-res"
-version = "0.4.0"
-source = "git+https://github.com/ruma/ruma?rev=5e574319f4e19d03898471e9792bdf5f89d94b26#5e574319f4e19d03898471e9792bdf5f89d94b26"
+version = "0.4.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "518c1afbddfcc5ffac8818a5cf0902709e6eca11aca8f24f6479df6f0601f1ba"
 dependencies = [
 "itertools 0.10.1",
 "js_int",
@@ -2461,9 +2476,9 @@ checksum = "2579985fda508104f7587689507983eadd6a6e84dd35d6d115361f530916fa0d"

 [[package]]
 name = "sha2"
-version = "0.9.5"
+version = "0.9.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b362ae5752fd2137731f9fa25fd4d9058af34666ca1966fb969119cc35719f12"
+checksum = "9204c41a1597a8c5af23c82d1c921cb01ec0a4c59e07a9c7306062829a3903f3"
 dependencies = [
 "block-buffer",
 "cfg-if 1.0.0",
@@ -18,8 +18,8 @@ edition = "2018"
 rocket = { version = "0.5.0-rc.1", features = ["tls"] } # Used to handle requests

 # Used for matrix spec type definitions and helpers
-#ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
-ruma = { git = "https://github.com/ruma/ruma", rev = "5e574319f4e19d03898471e9792bdf5f89d94b26", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+ruma = { version = "0.4.0", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
+#ruma = { git = "https://github.com/ruma/ruma", rev = "f5ab038e22421ed338396ece977b6b2844772ced", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { git = "https://github.com/timokoesters/ruma", rev = "50c1db7e0a3a21fc794b0cce3b64285a4c750c71", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }
 #ruma = { path = "../ruma/crates/ruma", features = ["compat", "rand", "appservice-api-c", "client-api", "federation-api", "push-gateway-api-c", "state-res", "unstable-pre-spec", "unstable-exhaustive-types"] }

@@ -50,6 +50,17 @@ If you want to connect an Appservice to Conduit, take a look at [APPSERVICES.md]
 3. Fork the repo and work on the issue. #conduit:nordgedanken.dev is happy to help :)
 4. Submit a MR

+
+#### Thanks to
+
+Thanks to Famedly, Prototype Fund (DLR and German BMBF) and all other individuals for financially supporting this project.
+
+Thanks to the contributors to Conduit and all libraries we use, for example:
+
+- Ruma: A clean library for the Matrix Spec in Rust
+- Rocket: A flexible web framework
+
+
 #### Donate

 Liberapay: <https://liberapay.com/timokoesters/>\
@@ -97,8 +97,8 @@ pub struct Rooms {
    pub(super) referencedevents: Arc<dyn Tree>,

    pub(super) pdu_cache: Mutex<LruCache<EventId, Arc<PduEvent>>>,
+    pub(super) shorteventid_cache: Mutex<LruCache<u64, Arc<EventId>>>,
    pub(super) auth_chain_cache: Mutex<LruCache<Vec<u64>, Arc<HashSet<u64>>>>,
-    pub(super) shorteventid_cache: Mutex<LruCache<u64, EventId>>,
    pub(super) eventidshort_cache: Mutex<LruCache<EventId, u64>>,
    pub(super) statekeyshort_cache: Mutex<LruCache<(EventType, String), u64>>,
    pub(super) shortstatekey_cache: Mutex<LruCache<u64, (EventType, String)>>,
@@ -121,7 +121,7 @@ impl Rooms {
    /// Builds a StateMap by iterating over all keys that start
    /// with state_hash, this gives the full state for the given state_hash.
    #[tracing::instrument(skip(self))]
-    pub fn state_full_ids(&self, shortstatehash: u64) -> Result<BTreeMap<u64, EventId>> {
+    pub fn state_full_ids(&self, shortstatehash: u64) -> Result<BTreeMap<u64, Arc<EventId>>> {
        let full_state = self
            .load_shortstatehash_info(shortstatehash)?
            .pop()
@@ -172,7 +172,7 @@ impl Rooms {
        shortstatehash: u64,
        event_type: &EventType,
        state_key: &str,
-    ) -> Result<Option<EventId>> {
+    ) -> Result<Option<Arc<EventId>>> {
        let shortstatekey = match self.get_shortstatekey(event_type, state_key)? {
            Some(s) => s,
            None => return Ok(None),
@@ -527,7 +527,7 @@ impl Rooms {
    pub fn parse_compressed_state_event(
        &self,
        compressed_event: CompressedStateEvent,
-    ) -> Result<(u64, EventId)> {
+    ) -> Result<(u64, Arc<EventId>)> {
        Ok((
            utils::u64_from_bytes(&compressed_event[0..size_of::<u64>()])
                .expect("bytes have right length"),
@@ -843,14 +843,14 @@ impl Rooms {
    }

    #[tracing::instrument(skip(self))]
-    pub fn get_eventid_from_short(&self, shorteventid: u64) -> Result<EventId> {
+    pub fn get_eventid_from_short(&self, shorteventid: u64) -> Result<Arc<EventId>> {
        if let Some(id) = self
            .shorteventid_cache
            .lock()
            .unwrap()
            .get_mut(&shorteventid)
        {
-            return Ok(id.clone());
+            return Ok(Arc::clone(id));
        }

        let bytes = self
@@ -858,15 +858,17 @@ impl Rooms {
            .get(&shorteventid.to_be_bytes())?
            .ok_or_else(|| Error::bad_database("Shorteventid does not exist"))?;

-        let event_id = EventId::try_from(utils::string_from_bytes(&bytes).map_err(|_| {
-            Error::bad_database("EventID in shorteventid_eventid is invalid unicode.")
-        })?)
-        .map_err(|_| Error::bad_database("EventId in shorteventid_eventid is invalid."))?;
+        let event_id = Arc::new(
+            EventId::try_from(utils::string_from_bytes(&bytes).map_err(|_| {
+                Error::bad_database("EventID in shorteventid_eventid is invalid unicode.")
+            })?)
+            .map_err(|_| Error::bad_database("EventId in shorteventid_eventid is invalid."))?,
+        );

        self.shorteventid_cache
            .lock()
            .unwrap()
-            .insert(shorteventid, event_id.clone());
+            .insert(shorteventid, Arc::clone(&event_id));

        Ok(event_id)
    }
@@ -933,7 +935,7 @@ impl Rooms {
        room_id: &RoomId,
        event_type: &EventType,
        state_key: &str,
-    ) -> Result<Option<EventId>> {
+    ) -> Result<Option<Arc<EventId>>> {
        if let Some(current_shortstatehash) = self.current_shortstatehash(room_id)? {
            self.state_get_id(current_shortstatehash, event_type, state_key)
        } else {
@@ -1541,7 +1543,7 @@ impl Rooms {
                                                let start = Instant::now();
                                                let count = server_server::get_auth_chain(
                                                    &room_id,
-                                                    vec![event_id],
+                                                    vec![Arc::new(event_id)],
                                                    db,
                                                )?
                                                .count();
@@ -1006,7 +1006,12 @@ pub(crate) async fn handle_incoming_pdu<'a>(
    // 9. Fetch any missing prev events doing all checks listed here starting at 1. These are timeline events
    let mut graph = HashMap::new();
    let mut eventid_info = HashMap::new();
-    let mut todo_outlier_stack = incoming_pdu.prev_events.clone();
+    let mut todo_outlier_stack = incoming_pdu
+        .prev_events
+        .iter()
+        .cloned()
+        .map(Arc::new)
+        .collect::<Vec<_>>();

    let mut amount = 0;

@@ -1025,7 +1030,7 @@ pub(crate) async fn handle_incoming_pdu<'a>(
            if amount > 100 {
                // Max limit reached
                warn!("Max prev event limit reached!");
-                graph.insert(prev_event_id.clone(), HashSet::new());
+                graph.insert((*prev_event_id).clone(), HashSet::new());
                continue;
            }

@@ -1042,46 +1047,47 @@ pub(crate) async fn handle_incoming_pdu<'a>(
                    amount += 1;
                    for prev_prev in &pdu.prev_events {
                        if !graph.contains_key(prev_prev) {
-                            todo_outlier_stack.push(dbg!(prev_prev.clone()));
+                            todo_outlier_stack.push(dbg!(Arc::new(prev_prev.clone())));
                        }
                    }

                    graph.insert(
-                        prev_event_id.clone(),
+                        (*prev_event_id).clone(),
                        pdu.prev_events.iter().cloned().collect(),
                    );
                    eventid_info.insert(prev_event_id.clone(), (pdu, json));
                } else {
                    // Time based check failed
-                    graph.insert(prev_event_id.clone(), HashSet::new());
+                    graph.insert((*prev_event_id).clone(), HashSet::new());
                    eventid_info.insert(prev_event_id.clone(), (pdu, json));
                }
            } else {
                // Get json failed
-                graph.insert(prev_event_id.clone(), HashSet::new());
+                graph.insert((*prev_event_id).clone(), HashSet::new());
            }
        } else {
            // Fetch and handle failed
-            graph.insert(prev_event_id.clone(), HashSet::new());
+            graph.insert((*prev_event_id).clone(), HashSet::new());
        }
    }

-    let sorted = state_res::lexicographical_topological_sort(dbg!(&graph), |event_id| {
-        // This return value is the key used for sorting events,
-        // events are then sorted by power level, time,
-        // and lexically by event_id.
-        println!("{}", event_id);
-        Ok((
-            0,
-            MilliSecondsSinceUnixEpoch(
-                eventid_info
-                    .get(event_id)
-                    .map_or_else(|| uint!(0), |info| info.0.origin_server_ts.clone()),
-            ),
-            ruma::event_id!("$notimportant"),
-        ))
-    })
-    .map_err(|_| "Error sorting prev events".to_owned())?;
+    let sorted =
+        state_res::StateResolution::lexicographical_topological_sort(dbg!(&graph), |event_id| {
+            // This return value is the key used for sorting events,
+            // events are then sorted by power level, time,
+            // and lexically by event_id.
+            println!("{}", event_id);
+            Ok((
+                0,
+                MilliSecondsSinceUnixEpoch(
+                    eventid_info
+                        .get(event_id)
+                        .map_or_else(|| uint!(0), |info| info.0.origin_server_ts),
+                ),
+                ruma::event_id!("$notimportant"),
+            ))
+        })
+        .map_err(|_| "Error sorting prev events".to_owned())?;

    let mut errors = 0;
    for prev_id in dbg!(sorted) {
@@ -1201,7 +1207,12 @@ fn handle_outlier_pdu<'a>(
        fetch_and_handle_outliers(
            db,
            origin,
-            &incoming_pdu.auth_events,
+            &incoming_pdu
+                .auth_events
+                .iter()
+                .cloned()
+                .map(Arc::new)
+                .collect::<Vec<_>>(),
            &create_event,
            &room_id,
            pub_key_map,
@@ -1270,7 +1281,7 @@ fn handle_outlier_pdu<'a>(
        if !state_res::event_auth::auth_check(
            &room_version,
            &incoming_pdu,
-            previous_create.clone(),
+            previous_create,
            None, // TODO: third party invite
            |k, s| auth_events.get(&(k.clone(), s.to_owned())).map(Arc::clone),
        )
@@ -1357,7 +1368,7 @@ async fn upgrade_outlier_to_timeline_pdu(
                    .get_or_create_shortstatekey(&prev_pdu.kind, state_key, &db.globals)
                    .map_err(|_| "Failed to create shortstatekey.".to_owned())?;

-                state.insert(shortstatekey, prev_event.clone());
+                state.insert(shortstatekey, Arc::new(prev_event.clone()));
                // Now it's the state after the pdu
            }

@@ -1400,7 +1411,7 @@ async fn upgrade_outlier_to_timeline_pdu(
                            .rooms
                            .get_or_create_shortstatekey(&prev_event.kind, state_key, &db.globals)
                            .map_err(|_| "Failed to create shortstatekey.".to_owned())?;
-                        leaf_state.insert(shortstatekey, prev_event.event_id.clone());
+                        leaf_state.insert(shortstatekey, Arc::new(prev_event.event_id.clone()));
                        // Now it's the state after the pdu
                    }

@@ -1421,17 +1432,17 @@ async fn upgrade_outlier_to_timeline_pdu(
                        db,
                    )
                    .map_err(|_| "Failed to load auth chain.".to_owned())?
-                    .map(|event_id| event_id.clone())
+                    .map(|event_id| (*event_id).clone())
                    .collect(),
                );
            }

-            let fork_states = fork_states
+            let fork_states = &fork_states
                .into_iter()
-                .map(|map| map.into_iter().map(|(k, id)| (k, id.clone())).collect())
+                .map(|map| map.into_iter().map(|(k, id)| (k, (*id).clone())).collect())
                .collect::<Vec<_>>();

-            state_at_incoming_event = match state_res::resolve(
+            state_at_incoming_event = match state_res::StateResolution::resolve(
                &room_id,
                room_version_id,
                &fork_states,
@@ -1452,7 +1463,7 @@ async fn upgrade_outlier_to_timeline_pdu(
                                .rooms
                                .get_or_create_shortstatekey(&event_type, &state_key, &db.globals)
                                .map_err(|_| "Failed to get_or_create_shortstatekey".to_owned())?;
-                            Ok((shortstatekey, event_id))
+                            Ok((shortstatekey, Arc::new(event_id)))
                        })
                        .collect::<StdResult<_, String>>()?,
                ),
@@ -1485,7 +1496,11 @@ async fn upgrade_outlier_to_timeline_pdu(
                let state_vec = fetch_and_handle_outliers(
                    &db,
                    origin,
-                    &res.pdu_ids,
+                    &res.pdu_ids
+                        .iter()
+                        .cloned()
+                        .map(Arc::new)
+                        .collect::<Vec<_>>(),
                    &create_event,
                    &room_id,
                    pub_key_map,
@@ -1506,7 +1521,7 @@ async fn upgrade_outlier_to_timeline_pdu(

                    match state.entry(shortstatekey) {
                        btree_map::Entry::Vacant(v) => {
-                            v.insert(pdu.event_id.clone());
+                            v.insert(Arc::new(pdu.event_id.clone()));
                        }
                        btree_map::Entry::Occupied(_) => return Err(
                            "State event's type and state_key combination exists multiple times."
@@ -1522,7 +1537,9 @@ async fn upgrade_outlier_to_timeline_pdu(
                    .map_err(|_| "Failed to talk to db.")?
                    .expect("Room exists");

-                if state.get(&create_shortstatekey) != Some(&create_event.event_id) {
+                if state.get(&create_shortstatekey).map(|id| id.as_ref())
+                    != Some(&create_event.event_id)
+                {
                    return Err("Incoming event refers to wrong create event.".to_owned());
                }

@@ -1717,7 +1734,7 @@ async fn upgrade_outlier_to_timeline_pdu(
                    .rooms
                    .get_or_create_shortstatekey(&leaf_pdu.kind, state_key, &db.globals)
                    .map_err(|_| "Failed to create shortstatekey.".to_owned())?;
-                leaf_state.insert(shortstatekey, leaf_pdu.event_id.clone());
+                leaf_state.insert(shortstatekey, Arc::new(leaf_pdu.event_id.clone()));
                // Now it's the state after the pdu
            }

@@ -1732,9 +1749,9 @@ async fn upgrade_outlier_to_timeline_pdu(
                .get_or_create_shortstatekey(&incoming_pdu.kind, state_key, &db.globals)
                .map_err(|_| "Failed to create shortstatekey.".to_owned())?;

-            state_after.insert(shortstatekey, incoming_pdu.event_id.clone());
+            state_after.insert(shortstatekey, Arc::new(incoming_pdu.event_id.clone()));
        }
-        fork_states.push(state_after.clone());
+        fork_states.push(state_after);

        let mut update_state = false;
        // 14. Use state resolution to find new room state
@@ -1764,7 +1781,7 @@ async fn upgrade_outlier_to_timeline_pdu(
                        db,
                    )
                    .map_err(|_| "Failed to load auth chain.".to_owned())?
-                    .map(|event_id| event_id.clone())
+                    .map(|event_id| (*event_id).clone())
                    .collect(),
                );
            }
@@ -1773,13 +1790,17 @@ async fn upgrade_outlier_to_timeline_pdu(
                .into_iter()
                .map(|map| {
                    map.into_iter()
-                        .map(|(k, id)| db.rooms.get_statekey_from_short(k).map(|k| (k, id.clone())))
+                        .map(|(k, id)| {
+                            db.rooms
+                                .get_statekey_from_short(k)
+                                .map(|k| (k, (*id).clone()))
+                        })
                        .collect::<Result<StateMap<_>>>()
                })
                .collect::<Result<Vec<_>>>()
                .map_err(|_| "Failed to get_statekey_from_short.".to_owned())?;

-            let state = match state_res::resolve(
+            let state = match state_res::StateResolution::resolve(
                &room_id,
                room_version_id,
                fork_states,
@@ -1858,7 +1879,7 @@ async fn upgrade_outlier_to_timeline_pdu(
 pub(crate) fn fetch_and_handle_outliers<'a>(
    db: &'a Database,
    origin: &'a ServerName,
-    events: &'a [EventId],
+    events: &'a [Arc<EventId>],
    create_event: &'a PduEvent,
    room_id: &'a RoomId,
    pub_key_map: &'a RwLock<BTreeMap<String, BTreeMap<String, String>>>,
@@ -1913,12 +1934,12 @@ pub(crate) fn fetch_and_handle_outliers<'a>(
                                match crate::pdu::gen_event_id_canonical_json(&res.pdu) {
                                    Ok(t) => t,
                                    Err(_) => {
-                                        back_off(id.clone());
+                                        back_off((**id).clone());
                                        continue;
                                    }
                                };

-                            if calculated_event_id != *id {
+                            if calculated_event_id != **id {
                                warn!("Server didn't return event id we requested: requested: {}, we got {}. Event: {:?}",
                                    id, calculated_event_id, &res.pdu);
                            }
@@ -1938,14 +1959,14 @@ pub(crate) fn fetch_and_handle_outliers<'a>(
                                Ok((pdu, json)) => (pdu, Some(json)),
                                Err(e) => {
                                    warn!("Authentication of event {} failed: {:?}", id, e);
-                                    back_off(id.clone());
+                                    back_off((**id).clone());
                                    continue;
                                }
                            }
                        }
                        Err(_) => {
                            warn!("Failed to fetch event: {}", id);
-                            back_off(id.clone());
+                            back_off((**id).clone());
                            continue;
                        }
                    }
@@ -2217,9 +2238,9 @@ fn append_incoming_pdu(
 #[tracing::instrument(skip(starting_events, db))]
 pub(crate) fn get_auth_chain<'a>(
    room_id: &RoomId,
-    starting_events: Vec<EventId>,
+    starting_events: Vec<Arc<EventId>>,
    db: &'a Database,
-) -> Result<impl Iterator<Item = EventId> + 'a> {
+) -> Result<impl Iterator<Item = Arc<EventId>> + 'a> {
    const NUM_BUCKETS: usize = 50;

    let mut buckets = vec![BTreeSet::new(); NUM_BUCKETS];
@@ -2495,12 +2516,12 @@ pub fn get_event_authorization_route(
        return Err(Error::BadRequest(ErrorKind::NotFound, "Event not found."));
    }

-    let auth_chain_ids = get_auth_chain(&room_id, vec![body.event_id.clone()], &db)?;
+    let auth_chain_ids = get_auth_chain(&room_id, vec![Arc::new(body.event_id.clone())], &db)?;

    Ok(get_event_authorization::v1::Response {
        auth_chain: auth_chain_ids
-            .filter_map(|id| Some(db.rooms.get_pdu_json(&id).ok()??))
-            .map(|event| PduEvent::convert_to_outgoing_federation_event(event))
+            .filter_map(|id| db.rooms.get_pdu_json(&id).ok()?)
+            .map(PduEvent::convert_to_outgoing_federation_event)
            .collect(),
    }
    .into())
@@ -2553,7 +2574,7 @@ pub fn get_room_state_route(
        })
        .collect();

-    let auth_chain_ids = get_auth_chain(&body.room_id, vec![body.event_id.clone()], &db)?;
+    let auth_chain_ids = get_auth_chain(&body.room_id, vec![Arc::new(body.event_id.clone())], &db)?;

    Ok(get_room_state::v1::Response {
        auth_chain: auth_chain_ids
@@ -2609,13 +2630,13 @@ pub fn get_room_state_ids_route(
        .rooms
        .state_full_ids(shortstatehash)?
        .into_iter()
-        .map(|(_, id)| id)
+        .map(|(_, id)| (*id).clone())
        .collect();

-    let auth_chain_ids = get_auth_chain(&body.room_id, vec![body.event_id.clone()], &db)?;
+    let auth_chain_ids = get_auth_chain(&body.room_id, vec![Arc::new(body.event_id.clone())], &db)?;

    Ok(get_room_state_ids::v1::Response {
-        auth_chain_ids: auth_chain_ids.collect(),
+        auth_chain_ids: auth_chain_ids.map(|id| (*id).clone()).collect(),
        pdu_ids,
    }
    .into())
Author	SHA1	Message	Date
Timo Kösters	73b764322f	Merge branch 'adjust-ci-for-next-and-master-development' into 'master' CI: Adjust CI for master and next branch development model (master) See merge request famedly/conduit!187	2021-09-02 18:25:57 +00:00
Jonas Zohren	a1f51440e2	chore(CI): Adjust CI for master and next branch development model - Build release builds for branches "master" and "next" - Push docker images under different tags, depending on why the pipeline started - branch master: push to `latest` - branch next: push to `next` - tag: push to `$TAG_NAME` Signed-off-by: Jonas Zohren <git-pbkyr@jzohren.de>	2021-09-02 17:17:51 +02:00
Timo Kösters	0d6ae2e1e7	Merge branch 'thanks' into 'master' docs: add thanks to See merge request famedly/conduit!185	2021-09-02 11:10:23 +00:00
Timo Kösters	d0baca44f8	docs: add thanks to	2021-09-02 13:05:19 +02:00