CI: Use curl instead of wget

The rust docker image already comes with curl, no need to install wget.
CI: Optionally use sccache for compilation
2021-12-14 11:16:40 +01:00 · 2021-12-14 11:16:02 +01:00 · 2021-11-25 22:36:44 +00:00 · 2021-11-25 22:36:44 +00:00
4 changed files with 58 additions and 82 deletions
@@ -26,16 +26,19 @@ variables:
  cache:
    paths:
      - cargohome
-      - target/
-    key: "build_cache--$TARGET--$CI_COMMIT_BRANCH--release"
+    key: "build_cache--$TARGET--$CI_COMMIT_BRANCH"
  variables:
    CARGO_PROFILE_RELEASE_LTO: "true"
    CARGO_PROFILE_RELEASE_CODEGEN_UNITS: "1"
+    CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
+    CARGO_HOME: $CI_PROJECT_DIR/cargohome
  before_script:
    - 'echo "Building for target $TARGET"'
-    - 'mkdir -p cargohome && CARGOHOME="cargohome"'
+    - "mkdir -p $CARGO_HOME"
    - "rustc --version && cargo --version && rustup show" # Print version info for debugging
    - "rustup target add $TARGET"
+    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
+    - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi
  script:
    - time cargo build --target $TARGET --release
    - 'cp "target/$TARGET/release/conduit" "conduit-$TARGET"'
@@ -142,8 +145,12 @@ build:debug:cargo:x86_64-unknown-linux-musl:
    DOCKER_HOST: tcp://docker:2375/
    DOCKER_TLS_CERTDIR: ""
    DOCKER_DRIVER: overlay2
-    PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64/v8,linux/amd64"
+    PLATFORMS: "linux/arm/v6,linux/arm/v7,linux/arm64,linux/amd64"
    DOCKER_FILE: "docker/ci-binaries-packaging.Dockerfile"
+  cache:
+    paths:
+      - docker_cache
+    key: "$CI_JOB_NAME"
  before_script:
    - docker login -u "$CI_REGISTRY_USER" -p "$CI_REGISTRY_PASSWORD" $CI_REGISTRY
    # Only log in to Dockerhub if the credentials are given:
@@ -156,80 +163,51 @@ build:debug:cargo:x86_64-unknown-linux-musl:
    - mkdir -p linux/ && mv ./conduit-x86_64-unknown-linux-musl linux/amd64
    - mkdir -p linux/arm/ && mv ./conduit-arm-unknown-linux-musleabihf linux/arm/v6
    - mkdir -p linux/arm/ && mv ./conduit-armv7-unknown-linux-musleabihf linux/arm/v7
-    - mkdir -p linux/arm64/ && mv ./conduit-aarch64-unknown-linux-musl linux/arm64/v8
-    # Actually create multiarch image:
+    - mv ./conduit-aarch64-unknown-linux-musl linux/arm64
+    - 'export CREATED=$(date -u +''%Y-%m-%dT%H:%M:%SZ'') && echo "Docker image creation date: $CREATED"'
+    # Build and push image:
    - >
      docker buildx build
      --pull
      --push
-      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
+      --cache-from=type=local,src=$CI_PROJECT_DIR/docker_cache
+      --cache-to=type=local,dest=$CI_PROJECT_DIR/docker_cache
+      --build-arg CREATED=$CREATED
      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
      --platform "$PLATFORMS"
-      --tag "$GL_IMAGE_TAG"
-      --tag "$GL_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA"
+      --tag "$TAG"
+      --tag "$TAG-alpine"
+      --tag "$TAG-commit-$CI_COMMIT_SHORT_SHA"
      --file "$DOCKER_FILE" .
-    # Only try to push to docker hub, if auth data for dockerhub exists:
-    - if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG"; fi
-    - if [ -n "${DOCKER_HUB}" ]; then docker push "$DH_IMAGE_TAG-commit-$CI_COMMIT_SHORT_SHA"; fi

-build:docker:next:
+docker:next:gitlab:
  extends: .docker-shared-settings
  rules:
    - if: '$CI_COMMIT_BRANCH == "next"'
  variables:
-    GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"
-    DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
+    TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:next"

-build:docker:master:
+docker:next:dockerhub:
+  extends: .docker-shared-settings
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "next" && $DOCKER_HUB'
+  variables:
+    TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
+
+docker:master:gitlab:
  extends: .docker-shared-settings
  rules:
    - if: '$CI_COMMIT_BRANCH == "master"'
  variables:
-    GL_IMAGE_TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"
-    DH_IMAGE_TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"
+    TAG: "$CI_REGISTRY_IMAGE/matrix-conduit:latest"

-## Build a docker image by packaging up the x86_64-unknown-linux-musl binary into an alpine image
-#.docker-shared-settings:
-#  stage: "build docker image"
-#  needs: []
-#  interruptible: true
-#  image:
-#    name: "gcr.io/kaniko-project/executor:debug"
-#    entrypoint: [""]
-#  tags: ["docker"]
-#  variables:
-#    # Configure Kaniko Caching: https://cloud.google.com/build/docs/kaniko-cache
-#    KANIKO_CACHE_ARGS: "--cache=true --cache-copy-layers=true --cache-ttl=120h --cache-repo $CI_REGISTRY_IMAGE/kaniko-ci-cache"
-#  before_script:
-#    - "mkdir -p /kaniko/.docker"
-#    - 'echo "{\"auths\":{\"$CI_REGISTRY\":{\"username\":\"$CI_REGISTRY_USER\",\"password\":\"$CI_REGISTRY_PASSWORD\"},\"$DOCKER_HUB\":{\"username\":\"$DOCKER_HUB_USER\",\"password\":\"$DOCKER_HUB_PASSWORD\"}}}" > /kaniko/.docker/config.json'
-#
-#
-#build:docker:next:
-#  extends: .docker-shared-settings
-#  needs:
-#    - "build:release:cargo:x86_64-unknown-linux-musl"
-#  script:
-#    - >
-#      /kaniko/executor
-#      $KANIKO_CACHE_ARGS
-#      --force
-#      --context $CI_PROJECT_DIR
-#      --build-arg CREATED=$(date -u +'%Y-%m-%dT%H:%M:%SZ')
-#      --build-arg VERSION=$(grep -m1 -o '[0-9].[0-9].[0-9]' Cargo.toml)
-#      --build-arg "GIT_REF=$CI_COMMIT_SHORT_SHA"
-#      --dockerfile "$CI_PROJECT_DIR/docker/ci-binaries-packaging.Dockerfile"
-#      --destination "$CI_REGISTRY_IMAGE/conduit:next"
-#      --destination "$CI_REGISTRY_IMAGE/conduit:next-alpine"
-#      --destination "$CI_REGISTRY_IMAGE/conduit:commit-$CI_COMMIT_SHORT_SHA"
-#      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next"
-#      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:next-alpine"
-#      --destination "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:commit-$CI_COMMIT_SHORT_SHA"
-#  rules:
-#    - if: '$CI_COMMIT_BRANCH == "next"'
-#
-#
+docker:master:dockerhub:
+  extends: .docker-shared-settings
+  rules:
+    - if: '$CI_COMMIT_BRANCH == "master" && $DOCKER_HUB'
+  variables:
+    TAG: "$DOCKER_HUB_IMAGE/matrixconduit/matrix-conduit:latest"

 # --------------------------------------------------------------------- #
 #  Run tests                                                            #
@@ -241,20 +219,21 @@ test:cargo:
  image: "rust:latest"
  tags: ["docker"]
  variables:
-    CARGO_HOME: "cargohome"
+    CARGO_HOME: "$CI_PROJECT_DIR/cargohome"
+    CARGO_INCREMENTAL: "false" # https://matklad.github.io/2021/09/04/fast-rust-builds.html#ci-workflow
  cache:
    paths:
-      - target
      - cargohome
-    key: test_cache
+    key: "test_cache--$CI_COMMIT_BRANCH"
  interruptible: true
  before_script:
-    - mkdir -p $CARGO_HOME && echo "using $CARGO_HOME to cache cargo deps"
+    - mkdir -p $CARGO_HOME
    - apt-get update -yqq
-    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config wget
+    - apt-get install -yqq --no-install-recommends build-essential libssl-dev pkg-config
    - rustup component add clippy rustfmt
-    - wget "https://faulty-storage.de/gitlab-report"
-    - chmod +x ./gitlab-report
+    - curl "https://faulty-storage.de/gitlab-report" --output ./gitlab-report && chmod +x ./gitlab-report
+    # If provided, bring in caching through sccache, which uses an external S3 endpoint to store compilation results:
+    - if [ -n "${SCCACHE_BIN_URL}" ]; then curl $SCCACHE_BIN_URL --output /sccache && chmod +x /sccache && export RUSTC_WRAPPER=/sccache; fi
  script:
    - rustc --version && cargo --version # Print version info for debugging
    - cargo fmt --all -- --check
@@ -12,20 +12,17 @@ only offer Linux binaries.

 You may simply download the binary that fits your machine. Run `uname -m` to see what you need. Now copy the right url:

-| CPU Architecture                            | Download link           |
-| ------------------------------------------- | ----------------------- |
-| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl] |
-| armv6                                       | [Download][armv6-musl]  |
-| armv7 (e.g. Raspberry Pi by default)        | [Download][armv7-musl]  |
-| armv8 / aarch64                             | [Download][armv8-musl]  |
+| CPU Architecture                            | Download stable version        |
+| ------------------------------------------- | ------------------------------ |
+| x84_64 / amd64 (Most servers and computers) | [Download][x84_64-musl-master] |
+| armv6                                       | [Download][armv6-musl-master]  |
+| armv7 (e.g. Raspberry Pi by default)        | [Download][armv7-musl-master]  |
+| armv8 / aarch64                             | [Download][armv8-musl-master]  |

-[x84_64-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
-
-[armv6-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
-
-[armv7-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
-
-[armv8-musl]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl
+[x84_64-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-x86_64-unknown-linux-musl?job=build:release:cargo:x86_64-unknown-linux-musl
+[armv6-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-arm-unknown-linux-musleabihf?job=build:release:cargo:arm-unknown-linux-musleabihf
+[armv7-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-armv7-unknown-linux-musleabihf?job=build:release:cargo:armv7-unknown-linux-musleabihf
+[armv8-musl-master]: https://gitlab.com/famedly/conduit/-/jobs/artifacts/master/raw/conduit-aarch64-unknown-linux-musl?job=build:release:cargo:aarch64-unknown-linux-musl

 ```bash
 $ sudo wget -O /usr/local/bin/matrix-conduit <url>
@@ -54,11 +54,11 @@ RUN apk add --no-cache \
 RUN mkdir -p /srv/conduit/.local/share/conduit

 # Test if Conduit is still alive, uses the same endpoint as Element
-COPY ./docker/healthcheck.sh /srv/conduit/
+COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
 HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh

 # Copy over the actual Conduit binary from the builder stage
-COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/
+COPY --from=builder /usr/src/conduit/target/release/conduit /srv/conduit/conduit

 # Improve security: Don't run stuff as root, that does not need to run as root:
 # Add www-data user and group with UID 82, as used by alpine
@@ -47,7 +47,7 @@ LABEL org.opencontainers.image.created=${CREATED} \
 RUN mkdir -p /srv/conduit/.local/share/conduit

 # Test if Conduit is still alive, uses the same endpoint as Element
-COPY ./docker/healthcheck.sh /srv/conduit/
+COPY ./docker/healthcheck.sh /srv/conduit/healthcheck.sh
 HEALTHCHECK --start-period=5s --interval=5s CMD ./healthcheck.sh
Author	SHA1	Message	Date
Jonas Zohren	adb518fa0d	CI: Use curl instead of wget The rust docker image already comes with curl, no need to install wget.	2021-12-14 11:16:40 +01:00
Jonas Zohren	f91216dd3c	CI: Optionally use sccache for compilation This moves compiler caching for incremental builds away from GitLab caching the whole target/ folder to caching each code unit in S3. This aleviates the need to zip and unzip and just caches on the fly. This feature is optional and gated behind the SCCACHE_BIN_URL env	2021-12-14 11:16:02 +01:00
Timo Kösters	ca724b6340	Merge branch '226-fix-docker-ci-issues' into 'next' Fixes for !225 See merge request famedly/conduit!226	2021-11-25 22:36:44 +00:00
Jonas Zohren	9bfc7b34b6	Fixes for !225	2021-11-25 22:36:44 +00:00