ci: refactor pipeline

The whole CI is refactored. The following problems are solved: 1. Couldn't test base image locally without overriding the old one. The testing stage is run in two ways depending on whether some of the docker-related files were changed. If they didn't (most likely), testing stage is run on the `latest` docker image as it used to. Otherwise, the docker image is rebuilt locally and tested. This is done by using `image:pull_policy` option, see [1]. It also makes a problem of using protected credentials irrelevant. 2. Gitlab caching used to be slow Stupid gitlab caching (see [2]) wasted on (un)zipping cache 4 minutes (of 6 total). Now it's replaced with a manual one: an archive (tar) is placed in a docker volume on a self-hosted gitlab runner with no compression. 3. Speed up fetching git. The tarantool-sys submodule requires a tag to be built correctly, but using shallow clones prevented that. We used to work-around it by specifying `GIT_DEPTH: 0`, but that's sub-optimal too. This patch introduces an elegant yet reliable solution: ``` until git describe; do git fetch --deepen 100; done ``` 4. No fine-grained time tracking was available Gitlab only shows the total time of `step_script` without splitting it into particular commands. This problem is solved by incorporating collapsible log sections feature, see [3]. It makes logs fancy too. Refs: - [1] https://docs.gitlab.com/ee/ci/yaml/#imagepull_policy - [2] https://gitlab.com/gitlab-org/gitlab-runner/-/issues/3523 - [3] https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections

ci: refactor pipeline
5d2dbc21 · Yaroslav Dynnikov · 265d3f82 · 5d2dbc21 · 5d2dbc21 · 5d2dbc21
Commit 5d2dbc21 authored 2 years ago by Yaroslav Dynnikov
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
 stages:
-  - prebuild
-  - build
+  - build-base-image
+  - test
+  - push-base-image
  - pack-centos
  - pack-ubuntu
  - pack-debian
@@ -8,85 +9,148 @@ stages:
  - deploy
  - test_rpm

-#workflow:
-#  rules:
-#    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
-#    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-#    - if: $CI_PIPELINE_SOURCE == "web"
+workflow:
+  # See https://docs.gitlab.com/ee/ci/jobs/job_control.html#avoid-duplicate-pipelines
+  rules:
+    # To avoid duplicate pipelines we disable merge request events,
+    # leaving only pushes and manual triggering.
+    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
+      when: never
+    - if: $CI_PIPELINE_SOURCE == "push"
+    - if: $CI_PIPELINE_SOURCE == "web"

 variables:
-  GIT_DEPTH: 0
-  GIT_SUBMODULE_STRATEGY: recursive
-  BASE_IMAGE: build-base:latest
-  CARGO_INCREMENTAL: 0
-  CARGO_HOME: ${CI_PROJECT_DIR}/.cargo
-  REGISTRY_IMAGE: docker-public.binary.picodata.io
-  RAW_REGISTRY: https://binary.picodata.io/repository/raw-private/
-
+  BASE_IMAGE: docker-public.binary.picodata.io/picodata-build-base
+  CARGO_HOME: /cache/.cargo
+  CACHE_PATHS: target .venv
+  CACHE_ARCHIVE: /cache/picodata.tar

-  #
-  # See also:
-  #   GitLab CI/CD predefined variables
-  #   https://docs.gitlab.com/ee/ci/variables/predefined_variables.html
-  #
-  # CI_COMMIT_SHA: The commit revision the project is built for
-  # CI_REGISTRY: The address of the GitLab Container Registry
-  # CI_REGISTRY_USER: The username to push containers to the project’s
-  #   GitLab Container Registry
-  # CI_REGISTRY_PASSWORD: The password to push containers to the project’s
-  #   GitLab Container Registry
-  # CI_REGISTRY_IMAGE: The address of the project’s Container Registry
-
-docker-build-base:
-  stage: prebuild
-  image: docker:20
+build-base-image:
+  stage: build-base-image
  tags:
-    - dind
+    - shell
  rules:
-    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
-      changes:
-        - docker-build-base/Dockerfile
-  before_script:
-    - mkdir -p $CI_PROJECT_DIR/.docker
-    - echo $DOCKER_AUTH_RW > $CI_PROJECT_DIR/.docker/config.json
+    - changes: &build-base-changes
+        paths:
+          - docker-build-base/**
+          - .gitlab-ci.yml
+        compare_to: master
+  variables:
+    GIT_DEPTH: 1
+    GIT_STRATEGY: fetch
+    GIT_SUBMODULE_STRATEGY: none
  script:
-    - docker pull ${REGISTRY_IMAGE}/${BASE_IMAGE} || true
+    - docker pull ${BASE_IMAGE}:latest || true
    - >
      docker build
-      --pull
+      --cache-from ${BASE_IMAGE}:latest
      --label GIT_COMMIT=${CI_COMMIT_SHA}
-      --cache-from ${REGISTRY_IMAGE}/${BASE_IMAGE}
-      -t ${REGISTRY_IMAGE}/${BASE_IMAGE}
+      -t ${BASE_IMAGE}:${CI_COMMIT_SHA}
      -f ./docker-build-base/Dockerfile
      ./docker-build-base
-    - docker --config $CI_PROJECT_DIR/.docker/ push ${REGISTRY_IMAGE}/${BASE_IMAGE}

-build:
-  image: ${REGISTRY_IMAGE}/${BASE_IMAGE}
-  stage: build
+push-base-image:
+  stage: push-base-image
+  tags:
+    - shell
+  rules:
+    - changes: *build-base-changes
+      if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
+  before_script:
+    - mkdir -p $CI_PROJECT_DIR/.docker
+    - echo $DOCKER_AUTH_RW > $CI_PROJECT_DIR/.docker/config.json
+  script:
+    - docker image tag ${BASE_IMAGE}:${CI_COMMIT_SHA} ${BASE_IMAGE}:latest
+    - docker --config $CI_PROJECT_DIR/.docker/ push ${BASE_IMAGE}:latest
+
+test:
+  extends: .test
+  rules:
+    # If some of docker files changed,
+    # then skip this job (and run
+    # test-base-image instead).
+    - changes: *build-base-changes
+      when: never
+    # else
+    - when: on_success
+  image:
+    name: ${BASE_IMAGE}:latest
+    pull_policy: if-not-present
+
+test-base-image:
+  extends: .test
+  rules:
+    - changes: *build-base-changes
+  image:
+    name: ${BASE_IMAGE}:${CI_COMMIT_SHA}
+    pull_policy: never
+
+.test:
+  stage: test
  tags:
    - docker
+  variables:
+    GIT_DEPTH: 100
+    GIT_SUBMODULE_STRATEGY: recursive
+    RUST_BACKTRACE: 1
+  before_script:
+    # Gitlab CI implicitly clones specific refs (e.g. `refs/pipelines/xxxxxxx`),
+    # but it doesn't imply fetching tags. We clone them manually with the
+    # `git fetch` command.
+    #
+    # Tags in `tarantool-sys` and `luajit` submodules are necessary for
+    # the build scripts. Without them the job fails.
+    - |
+      # Fetch tags
+      ci-log-section start "fetch-submodule-tags" Fetching tags for submodules
+      for s in tarantool-sys tarantool-sys/third_party/luajit; do
+        echo "Fetching tag for $s"
+        pushd $s
+        until git describe; do git fetch --deepen 100; done
+        popd
+      done
+      ci-log-section end "fetch-submodule-tags"
+
+    # Gitlab CI caching is shit. So we implement it manually
+    - |
+      # Restore cache
+      if [ -f "${CACHE_ARCHIVE}" ]; then
+        ci-log-section start "restore-cache" Restoring cache from ${CACHE_ARCHIVE} ...
+        tar -xf ${CACHE_ARCHIVE}
+        echo "Ok"
+        du -sh ${CACHE_PATHS} || true
+        ci-log-section end "restore-cache"
+      else
+        echo "No cache found"
+      fi
  script:
-    # - git submodule foreach --recursive 'git fetch --tags; git describe --always'
-    - pushd tarantool-sys; git describe; popd
-    - pushd tarantool-sys/third_party/luajit; git describe; popd
-    - export RUST_BACKTRACE=1
    - cargo -V
    - cargo build --locked
    - cargo test --locked
    - cargo fmt -- -v --check
    - cargo clippy --version
    - cargo clippy -- --deny clippy::all
-    - PIPENV_VENV_IN_PROJECT=1 PIP_NO_CACHE_DIR=true python3.10 -m pipenv install --deploy
+    - |
+      # Pipenv install
+      ci-log-section start "pipenv-install" Installing pip dependencies ...
+      PIPENV_VENV_IN_PROJECT=1 PIP_NO_CACHE_DIR=true python3.10 -m pipenv install --deploy
+      ci-log-section end "pipenv-install"
    - pipenv run pytest --numprocesses 2 -v
    - pipenv run lint
-  cache:
-    paths:
-      - target
-      - .cargo
-      - .venv
+    - |
+      # Save cache
+      if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
+        ci-log-section start "save-cache" Saving cache to ${CACHE_ARCHIVE} ...
+        du -sh ${CACHE_PATHS} || true
+        tar -cf "${CACHE_ARCHIVE}" ${CACHE_PATHS}
+        echo Ok
+        du -sh ${CACHE_ARCHIVE}
+        ci-log-section end "save-cache"
+      else
+        echo "Skip saving cache on a non-master branch"
+      fi

-build-package-centos:
+pack-centos:
  stage: pack-centos
  tags:
    - shell
@@ -107,7 +171,7 @@ build-package-centos:
      - build/picodata*.rpm
  dependencies: []

-build-package-ubuntu:
+pack-ubuntu:
  tags:
    - shell
  stage: pack-ubuntu
@@ -127,7 +191,7 @@ build-package-ubuntu:
      - build_ubuntu/*.deb
  dependencies: []

-build-package-debian:
+pack-debian:
  tags:
    - shell
  stage: pack-debian
@@ -166,7 +230,7 @@ sign-rpm-packages:
    paths:
      - build/picodata*.rpm
  dependencies:
-    - build-package-centos
+    - pack-centos

 deploy-job:
  stage: deploy
@@ -205,8 +269,8 @@ deploy-job:
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/debian/ -C main includedeb bullseye ~/.deb/debian/picodata*bullseye*deb; rm ~/.deb/debian/picodata*bullseye*deb"
    - echo "debian packets successfully deployed."
  dependencies:
-    - build-package-ubuntu
-    - build-package-debian
+    - pack-ubuntu
+    - pack-debian
    - sign-rpm-packages

 test_el7:

--- a/docker-build-base/Dockerfile
+++ b/docker-build-base/Dockerfile
@@ -21,8 +21,9 @@ RUN set -e; \
 RUN set -e; \
    curl --proto '=https' --tlsv1.2 -sSf https://sh.rustup.rs | \
    sh -s -- -y --profile default --default-toolchain 1.64.0
+ENV PATH=/root/.cargo/bin:${PATH}

-ENV PATH=/root/.rocks/bin:${PATH}
+COPY ci-log-section /usr/bin/ci-log-section

 # install Pipenv
 COPY requirements-pipenv.txt /tmp/requirements-pipenv.txt

--- a/docker-build-base/ci-log-section
+++ b/docker-build-base/ci-log-section
+#!/bin/bash
+
+usage() {
+    echo "Usage:"
+    echo "   ci-log-section start SECTION_NAME SECTION_HEADER"
+    echo "   ci-log-section end SECTION_NAME"
+    exit -1
+}
+
+# See: https://docs.gitlab.com/ee/ci/jobs/#custom-collapsible-sections
+
+if [ "$1" == "start" ]; then
+    [ -n "$2" ] || usage
+    echo -en "\e[0K"
+    echo -en "section_start:$(date +%s):$2[collapsed=true]"
+    echo -en "\r\e[0K"
+    echo "${@:3}"
+elif [ "$1" == "end" ]; then
+    [ -n "$2" ] || usage
+    echo -en "\e[0K"
+    echo -en "section_end:$(date +%s):$2"
+    echo -en "\r\e[0K"
+    echo
+else
+    usage
+fi