WIP: Serve static element web

Fixes for !225

See merge request famedly/conduit!226

.gitlab-ci.yml

@@ -142,8 +142,12 @@ build:debug:cargo:x86_64-unknown-linux-musl:
     DOCKER_HOST: tcp://docker:2375/
     DOCKER_TLS_CERTDIR: ""
     DOCKER_DRIVER: overlay2
-    PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/amd64"
+    PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
     DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
+  cache:
+    paths:
+      - docker_cache
+    key: "$CI_JOB_NAME"
   before_script:
     - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
     # Only log in to Dockerhub if the credentials are given:
@@ -156,80 +160,51 @@ build:debug:cargo:x86_64-unknown-linux-musl:
     - mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64
     - mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6
     - mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7
-    - mkdir -p linux/arm64/ && mv ./conduit-aarch64-unknown-linux-musl linux/arm64/v8
+    - mv ./conduit-aarch64-unknown-linux-musl linux/arm64
-    # Actually create multiarch image:
+    - 'export CREATED=$(date -u +''%Y-%m-%dT%H:%M:%SZ'') && echo "Docker image creation date: $CREATED"'
+    # Build and push image:
     - >
       docker buildx build
       --pull
       --push
-      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --cache-from=type=local,src=$CI_PROJECT_DIR/docker_cache
+      --cache-to=type=local,dest=$CI_PROJECT_DIR/docker_cache
+      --build-arg CREATED=$CREATED
       --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
       --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
       --platform "$PLATFORMS"
-      --tag "$GL_IMAGE_TAG"
+      --tag "$TAG"
-      --tag "$GL_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA"
+      --tag "$TAG-alpine"
+      --tag "$TAG-commit-$CI_COMMIT_SHORT_SHA"
       --file "$DOCKER_FILE" .
-    # Only try to push to docker hub, if auth data for dockerhub exists:
-    - if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG"; fi
-    - if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA"; fi
 
-build:docker:next:
+docker:next:gitlab:
   extends: .docker-shared-settings
   rules:
     - if: '$CI_COMMIT_BRANCH == "next"'
   variables:
-    GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"
+    TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"
-    DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
 
-build:docker:master:
+docker:next:dockerhub:
+  extends: .docker-shared-settings
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "next" && $DOCKER_HUB'
+  variables:
+    TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
+
+docker:master:gitlab:
   extends: .docker-shared-settings
   rules:
     - if: '$CI_COMMIT_BRANCH == "master"'
   variables:
-    GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"
+    TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"
-    DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
 
-## Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
+docker:master:dockerhub:
-#.docker-shared-settings:
+  extends: .docker-shared-settings
-#  stage: "build docker image"
+  rules:
-#  needs: []
+    - if: '$CI_COMMIT_BRANCH == "master" && $DOCKER_HUB'
-#  interruptible: true
+  variables:
-#  image:
+    TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
-#    name: "gcr.io/kaniko-project/executor:debug"
-#    entrypoint: [""]
-#  tags: ["docker"]
-#  variables:
-#    # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache
-#    KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache"
-#  before_script:
-#    - "mkdir -p /kaniko/.docker"
-#    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'
-#
-#
-#build:docker:next:
-#  extends: .docker-shared-settings
-#  needs:
-#    - "build:release:cargo:x86_64-unknown-linux-musl"
-#  script:
-#    - >
-#      /kaniko/executor
-#      $KANIKO_CACHE_ARGS
-#      --force
-#      --context $CI_PROJECT_DIR
-#      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-#      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
-#      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
-#      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
-#      --destination "$CI_REGISTRY_IMAGE/conduit:next"
-#      --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine"
-#      --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
-#      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
-#      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine"
-#      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
-#  rules:
-#    - if: '$CI_COMMIT_BRANCH == "next"'
-#
-#
 
 # --------------------------------------------------------------------- #
 #  Run tests                                                            #

Cargo.toml

@@ -82,6 +82,9 @@ thread_local = "1.1.3"
 # used for TURN server authentication
 hmac = "0.11.0"
 sha-1 = "0.9.8"
+# used to embed Element Web into Conduit
+rust-embed="6.3.0"
+
 
 [features]
 default = ["conduit_bin", "backend_sqlite"]

DEPLOY.md

@@ -12,20 +12,17 @@ only offer Linux binaries.
 
 You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:
 
-| CPU Architecture                            | Download link           |
+| CPU Architecture                            | Download stable version        |
-| ------------------------------------------- | ----------------------- |
+| ------------------------------------------- | ------------------------------ |
-| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl] |
+| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl-master] |
-| armv6                                       | [Download][armv6-musl]  |
+| armv6                                       | [Download][armv6-musl-master]  |
-| armv7 (e.g. Raspberry Pi by default)        | [Download][armv7-musl]  |
+| armv7 (e.g. Raspberry Pi by default)        | [Download][armv7-musl-master]  |
-| armv8 / aarch64                             | [Download][armv8-musl]  |
+| armv8 / aarch64                             | [Download][armv8-musl-master]  |
 
-[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
+[x84_64-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
-
+[armv6-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
-[armv6-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
+[armv7-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
-
+[armv8-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl
-[armv7-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
-
-[armv8-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl
 
 ```bash
 $ sudo wget -O /usr/local/bin/matrix-conduit <url>

Dockerfile

@@ -54,11 +54,11 @@ RUN apk add --no-cache \
 RUN mkdir -p /srv/conduit/.local/share/conduit
 
 # Test if Conduit is still alive, uses the same endpoint as Element
-COPY ./docker/healthcheck.sh /srv/conduit/
+COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
 HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
 
 # Copy over the actual Conduit binary from the builder stage
-COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/
+COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit
 
 # Improve security: Don't run stuff as root, that does not need to run as root:
 # Add www-data user and group with UID 82, as used by alpine

docker/ci-binaries-packaging.Dockerfile

@@ -47,7 +47,7 @@ LABEL org.opencontainers.image.created=${CREATED} \
 RUN mkdir -p /srv/conduit/.local/share/conduit
 
 # Test if Conduit is still alive, uses the same endpoint as Element
-COPY ./docker/healthcheck.sh /srv/conduit/
+COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
 HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
 
 

src/database.rs

@@ -314,11 +314,11 @@ impl Database {
                         .try_into()
                         .expect("pdu cache capacity fits into usize"),
                 )),
-                auth_chain_cache: Mutex::new(LruCache::new(10_000_000)),
+                auth_chain_cache: Mutex::new(LruCache::new(1_000_000)),
-                shorteventid_cache: Mutex::new(LruCache::new(10_000_000)),
+                shorteventid_cache: Mutex::new(LruCache::new(1_000_000)),
-                eventidshort_cache: Mutex::new(LruCache::new(10_000_000)),
+                eventidshort_cache: Mutex::new(LruCache::new(1_000_000)),
-                shortstatekey_cache: Mutex::new(LruCache::new(10_000_000)),
+                shortstatekey_cache: Mutex::new(LruCache::new(1_000_000)),
-                statekeyshort_cache: Mutex::new(LruCache::new(10_000_000)),
+                statekeyshort_cache: Mutex::new(LruCache::new(1_000_000)),
                 our_real_users_cache: RwLock::new(HashMap::new()),
                 appservice_in_room_cache: RwLock::new(HashMap::new()),
                 stateinfo_cache: Mutex::new(LruCache::new(1000)),

src/database/abstraction/sqlite.rs

@@ -21,7 +21,7 @@ thread_local! {
 
 struct PreparedStatementIterator<'a> {
     pub iterator: Box<dyn Iterator<Item = TupleOfBytes> + 'a>,
-    pub statement_ref: NonAliasingBox<rusqlite::CachedStatement<'a>>,
+    pub statement_ref: NonAliasingBox<rusqlite::Statement<'a>>,
 }
 
 impl Iterator for PreparedStatementIterator<'_> {
@@ -52,7 +52,7 @@ impl Engine {
     fn prepare_conn(path: &Path, cache_size_kb: u32) -> Result<Connection> {
         let conn = Connection::open(&path)?;
 
-        conn.pragma_update(Some(Main), "page_size", &32768)?;
+        conn.pragma_update(Some(Main), "page_size", &2048)?;
         conn.pragma_update(Some(Main), "journal_mode", &"WAL")?;
         conn.pragma_update(Some(Main), "synchronous", &"NORMAL")?;
         conn.pragma_update(Some(Main), "cache_size", &(-i64::from(cache_size_kb)))?;
@@ -136,7 +136,7 @@ impl SqliteTable {
     fn get_with_guard(&self, guard: &Connection, key: &[u8]) -> Result<Option<Vec<u8>>> {
         //dbg!(&self.name);
         Ok(guard
-            .prepare_cached(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())?
+            .prepare(format!("SELECT value FROM {} WHERE key = ?", self.name).as_str())?
             .query_row([key], |row| row.get(0))
             .optional()?)
     }
@@ -161,7 +161,7 @@ impl SqliteTable {
     ) -> Box<dyn Iterator<Item = TupleOfBytes> + 'a> {
         let statement = Box::leak(Box::new(
             guard
-                .prepare_cached(&format!(
+                .prepare(&format!(
                     "SELECT key, value FROM {} ORDER BY key ASC",
                     &self.name
                 ))
@@ -290,7 +290,7 @@ impl Tree for SqliteTable {
         if backwards {
             let statement = Box::leak(Box::new(
                 guard
-                    .prepare_cached(&format!(
+                    .prepare(&format!(
                         "SELECT key, value FROM {} WHERE key <= ? ORDER BY key DESC",
                         &self.name
                     ))
@@ -315,7 +315,7 @@ impl Tree for SqliteTable {
         } else {
             let statement = Box::leak(Box::new(
                 guard
-                    .prepare_cached(&format!(
+                    .prepare(&format!(
                         "SELECT key, value FROM {} WHERE key >= ? ORDER BY key ASC",
                         &self.name
                     ))

src/main.rs

@@ -17,11 +17,15 @@ mod pdu;
 mod ruma_wrapper;
 mod utils;
 
+use std::ffi::OsStr;
+use std::io::Cursor;
+use std::path::PathBuf;
 use std::sync::Arc;
 
 use database::Config;
 pub use database::Database;
 pub use error::{Error, Result};
+use http::StatusCode;
 use opentelemetry::trace::{FutureExt, Tracer};
 pub use pdu::PduEvent;
 pub use rocket::State;
@@ -34,8 +38,12 @@ use rocket::{
         providers::{Env, Format, Toml},
         Figment,
     },
-    routes, Request,
+    get,
+    http::{ContentType, Status},
+    response, routes, Request,
 };
+
+use rust_embed::RustEmbed;
 use tokio::sync::RwLock;
 use tracing_subscriber::{prelude::*, EnvFilter};
 
@@ -176,6 +184,7 @@ fn setup_rocket(config: Figment, data: Arc<RwLock<Database>>) -> rocket::Rocket<
                 server_server::claim_keys_route,
             ],
         )
+        .mount("/", routes![dist])
         .register(
             "/",
             catchers![
@@ -276,6 +285,34 @@ async fn main() {
     }
 }
 
+#[derive(RustEmbed)]
+#[folder = "element_web"]
+struct ElementWebAsset;
+
+#[get("/<file..>")]
+fn dist<'r>(file: PathBuf) -> response::Result<'r> {
+    let filename = file.display().to_string();
+
+    ElementWebAsset::get(&filename).map_or_else(
+        || Err(Status::NotFound),
+        |embedded_file| {
+            let ext = file
+                .as_path()
+                .extension()
+                .and_then(OsStr::to_str)
+                .ok_or_else(|| Status::new(400))?;
+
+            let content_type = ContentType::from_extension(ext).ok_or_else(|| Status::new(400))?;
+
+            let response: Result<response::Response<'r>, Status> = response::Response::build()
+                .header(content_type)
+                .sized_body(embedded_file.data.len(), Cursor::new(embedded_file.data))
+                .ok();
+            return response;
+        },
+    )
+}
+
 #[catch(404)]
 fn not_found_catcher(_: &Request<'_>) -> String {
     "404 Not Found".to_owned()

src/server_server.rs

@@ -47,6 +47,7 @@ use ruma::{
         },
         AnyEphemeralRoomEvent, EventType,
     },
+    int,
     receipt::ReceiptType,
     serde::JsonObject,
     signatures::{CanonicalJsonObject, CanonicalJsonValue},
@@ -750,14 +751,14 @@ pub async fn send_transaction_message_route(
                 .roomid_mutex_federation
                 .write()
                 .unwrap()
-                .entry(RoomId::try_from("!asdf:asfd.asdf").unwrap())
+                .entry(room_id.clone())
                 .or_default(),
         );
         let mutex_lock = mutex.lock().await;
         let start_time = Instant::now();
         resolved_map.insert(
             event_id.clone(),
-            dbg!(handle_incoming_pdu(
+            handle_incoming_pdu(
                 &body.origin,
                 &event_id,
                 &room_id,
@@ -766,7 +767,7 @@ pub async fn send_transaction_message_route(
                 &db,
                 &pub_key_map,
             )
-            .await)
+            .await
             .map(|_| ()),
         );
         drop(mutex_lock);
@@ -955,7 +956,6 @@ pub(crate) async fn handle_incoming_pdu<'a>(
     db: &'a Database,
     pub_key_map: &'a RwLock<BTreeMap<String, BTreeMap<String, String>>>,
 ) -> Result<Option<Vec<u8>>, String> {
-    warn!("Handling incoming pdu: {:?}", value);
     match db.rooms.exists(room_id) {
         Ok(true) => {}
         _ => {
@@ -1000,7 +1000,6 @@ pub(crate) async fn handle_incoming_pdu<'a>(
         return Ok(None);
     }
 
-    /* stop fetching prev events for now
     // 9. Fetch any missing prev events doing all checks listed here starting at 1. These are timeline events
     let mut graph = HashMap::new();
     let mut eventid_info = HashMap::new();
@@ -1115,7 +1114,6 @@ pub(crate) async fn handle_incoming_pdu<'a>(
             );
         }
     }
-    */
 
     upgrade_outlier_to_timeline_pdu(
         incoming_pdu,
@@ -2840,7 +2838,7 @@ async fn create_join_event(
             .roomid_mutex_federation
             .write()
             .unwrap()
-            .entry(RoomId::try_from("!asdf:asfd.asdf").unwrap()) // make all rooms share the same lock for now
+            .entry(room_id.clone())
             .or_default(),
     );
     let mutex_lock = mutex.lock().await;