.gitlab-ci.yml

stages:
  - build-base-image
  - test
  - pack
  - sign
  - deploy
  - check-deployment

workflow:
  # See https://docs.gitlab.com/ee/ci/jobs/job_control.html#avoid-duplicate-pipelines
  rules:
    # To avoid duplicate pipelines we disable merge request events,
    # leaving only pushes and manual triggering.
    - if: $CI_PIPELINE_SOURCE == "merge_request_event"
      when: never
    - if: $CI_PIPELINE_SOURCE == "push"
    - if: $CI_PIPELINE_SOURCE == "web"

variables:
  REGISTRY: docker-public.binary.picodata.io
  BASE_IMAGE: ${REGISTRY}/picodata-build-base
  CARGO_HOME: /shared-storage/picodata/.cargo
  CACHE_PATHS: target .venv
  CACHE_ARCHIVE: /shared-storage/picodata/cache.tar

# job:rules explained:
#
# - if build-base changes on master branch (compared to HEAD~1)
#     * build-base-image (with tag latest) and push
#     * test (on base-image:latest)
# - if build-base changes on development branch (compared to master)
#     * build-base-image (with tag sha)
#     * test (on base-image:sha)
# - else (if build-base doesn't change)
#     * skip build-base-image
#     * just test (on base-image:latest)
#
# Anchor syntax explained here:
# https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html
#
.rules:
  - &if-build-base-changes-on-master-branch
    if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
    changes:
      # implies compare_to HEAD~1
      paths: &build-base-changes-paths
        - docker-build-base/**
        - .gitlab-ci.yml

  - &if-build-base-changes-on-dev-branch
    if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
    changes:
      compare_to: master
      paths: *build-base-changes-paths

  - &else {}

build-base-image:
  stage: build-base-image
  tags:
    - shell-yc
  rules:
    - <<: *if-build-base-changes-on-master-branch
      variables:
        BASE_IMAGE_TAG: latest
    - <<: *if-build-base-changes-on-dev-branch
      variables:
        BASE_IMAGE_TAG: ${CI_COMMIT_SHA}
    - <<: *else
      when: never
  variables:
    GIT_DEPTH: 1
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: none
  script:
    - docker pull ${BASE_IMAGE}:latest || true
    - >
      docker build
      --cache-from ${BASE_IMAGE}:latest
      --label GIT_COMMIT=${CI_COMMIT_SHA}
      -t ${BASE_IMAGE}:${BASE_IMAGE_TAG}
      -f ./docker-build-base/Dockerfile
      ./docker-build-base
    - |
      # Push image to registry
      if [ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]; then
        echo "Pushing ${BASE_IMAGE}:${BASE_IMAGE_TAG}"
        mkdir -p $CI_PROJECT_DIR/.docker
        echo $DOCKER_AUTH_RW > $CI_PROJECT_DIR/.docker/config.json
        docker --config $CI_PROJECT_DIR/.docker/ push ${BASE_IMAGE}:${BASE_IMAGE_TAG}
      else
        echo "Skip pushing image on a non-master branch"
      fi

test:
  stage: test
  tags:
    - docker-yc
  rules:
    - <<: *if-build-base-changes-on-master-branch
      variables:
        BASE_IMAGE_TAG: latest
    - <<: *if-build-base-changes-on-dev-branch
      variables:
        BASE_IMAGE_TAG: ${CI_COMMIT_SHA}
    - <<: *else
      variables:
        BASE_IMAGE_TAG: latest
  image:
    name: ${BASE_IMAGE}:${BASE_IMAGE_TAG}
    pull_policy: if-not-present
  variables:
    GIT_DEPTH: 100
    GIT_SUBMODULE_STRATEGY: recursive
    RUST_BACKTRACE: full
  before_script:
    # Gitlab CI implicitly clones specific refs (e.g. `refs/pipelines/xxxxxxx`),
    # but it doesn't imply fetching tags. We clone them manually with the
    # `git fetch` command.
    #
    # Tags in `tarantool-sys` and `luajit` submodules are necessary for
    # the build scripts. Without them the job fails.
    - &fetch-tags |
      # Fetch tags
      ci-log-section start "fetch-submodule-tags" Fetching tags for submodules
      for s in tarantool-sys tarantool-sys/third_party/luajit; do
        echo "Fetching tag for $s"
        pushd $s
        until git describe; do git fetch --deepen 100; done
        popd
      done
      ci-log-section end "fetch-submodule-tags"

    # Gitlab CI caching is shit. So we implement it manually
    - |
      # Restore cache
      if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
        echo "Skip restoring cache on the master branch"
      elif [ -f "${CACHE_ARCHIVE}" ]; then
        ci-log-section start "restore-cache" Restoring cache from ${CACHE_ARCHIVE} ...
        tar -xf ${CACHE_ARCHIVE}
        echo "Ok"
        du -sh ${CACHE_PATHS} || true
        ci-log-section end "restore-cache"
      else
        echo "No cache found"
      fi
  script:
    - cargo -V
    - cargo build --locked
    - cargo test --locked
    - cargo fmt -- -v --check
    - cargo clippy --version
    - cargo clippy -- --deny clippy::all
    - |
      # Pipenv install
      ci-log-section start "pipenv-install" Installing pip dependencies ...
      PIPENV_VENV_IN_PROJECT=1 PIP_NO_CACHE_DIR=true python3.10 -m pipenv install --deploy
      ci-log-section end "pipenv-install"
    - pipenv run pytest --numprocesses auto -v
    - pipenv run lint
    - |
      # Save cache
      if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then
        ci-log-section start "save-cache" Saving cache to ${CACHE_ARCHIVE} ...
        du -sh ${CACHE_PATHS} || true
        TMPEXT=$RANDOM
        tar -cf "${CACHE_ARCHIVE}.${TMPEXT}" ${CACHE_PATHS}
        mv -f "${CACHE_ARCHIVE}.${TMPEXT}" "${CACHE_ARCHIVE}"
        echo Ok
        du -sh ${CACHE_ARCHIVE}
        ci-log-section end "save-cache"
      else
        echo "Skip saving cache on a non-master branch"
      fi

test-docker:
  stage: test
  tags:
    - shell
  rules:
    - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH
      changes:
        compare_to: master
        paths:
          - helm/picodata.Dockerfile
          - helm/picodata-diag.Dockerfile
    - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH
      when: on_success
    - # else
      when: never
  variables:
    GIT_DEPTH: 100
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: recursive
  before_script:
    - export PATH=docker-build-base:$PATH
    - *fetch-tags
  script:
    - |
      # Build docker images
      for image in picodata picodata-diag; do
        ci-log-section start "test-docker-${image}" Building docker image ${image}
        docker build \
          --label GIT_COMMIT=${CI_COMMIT_SHA} \
          -f helm/${image}.Dockerfile .
        ci-log-section end "test-docker-${image}"
      done

.pack:
  tags:
    - shell-limit-1
  only:
    - web
    - tags
  variables:
    PRESERVE_ENVVARS: VER_TNT
    GIT_DEPTH: 100
    GIT_SUBMODULE_STRATEGY: recursive
  before_script:
    - git clone https://github.com/packpack/packpack.git packpack
    - |
      # Describe tarantool-sys
      pushd tarantool-sys
      until git describe; do git fetch --deepen 100; done
      export VER_TNT=$(
        git describe --long |
        sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p'
      );
      echo $VER_TNT
      popd
    - |
      # Describe picodata
      until git describe; do git fetch --deepen 100; done
      git describe --long
  timeout: 2h
  dependencies: []

pack-centos:
  stage: pack
  extends: .pack
  script:
    - sed -i "s/(id -u)/(id -u) -o/g" packpack/packpack
    - OS=centos DIST=7 BUILDDIR=$PWD/build_centos packpack/packpack
    - OS=centos DIST=8 BUILDDIR=$PWD/build_centos packpack/packpack
  artifacts:
    paths:
      - build_centos/picodata*.rpm

pack-ubuntu:
  stage: pack
  extends: .pack
  script:
    - OS=ubuntu DIST=focal BUILDDIR=$PWD/build_${DIST}/ RELEASE=${DIST} packpack/packpack
    - OS=ubuntu DIST=jammy BUILDDIR=$PWD/build_${DIST}/ RELEASE=${DIST} packpack/packpack
  artifacts:
    paths:
      - build_focal/*.deb
      - build_jammy/*.deb

pack-debian:
  stage: pack
  extends: .pack
  script:
    - OS=debian DIST=bullseye BUILDDIR=$PWD/build_debian/ RELEASE=${DIST} packpack/packpack
  artifacts:
    paths:
      - build_debian/*.deb

pack-altlinux:
  stage: pack
  extends: .pack
  script:
    - DOCKER_REPO=docker-picodata.binary.picodata.io/packpack/alt DOCKER_IMAGE=p10 BUILDDIR=$PWD/build_alt packpack/packpack
    - DOCKER_REPO=docker-picodata.binary.picodata.io/packpack/alt DOCKER_IMAGE=p9 BUILDDIR=$PWD/build_alt packpack/packpack
  artifacts:
    paths:
      - build_alt/picodata*.rpm

pack-redos:
  stage: pack
  extends: .pack
  script:
    - OS=redos DIST=7.3 BUILDDIR=$PWD/build_redos/ packpack/packpack
  artifacts:
    paths:
      - build_redos/picodata*.rpm

pack-astralinux:
  stage: pack
  extends: .pack
  script:
    - DOCKER_REPO=docker-picodata.binary.picodata.io/packpack/astra DOCKER_IMAGE=orel-2.12 BUILDDIR=$PWD/build_astra packpack/packpack
  artifacts:
    paths:
      - build_astra/*.deb

sign-rpm-packages:
  variables:
    DOCKER_AUTH_CONFIG: $DOCKER_AUTH_RO
  stage: sign
  tags:
    - shell
  only:
    - web
    - tags
  before_script:
    - echo "$GPG_KEY_KDY" | base64 -d > build_centos/kdy.asc
    - echo "$GPG_KEY_KDY" | base64 -d > build_alt/kdy.asc
    - echo "$GPG_KEY_KDY" | base64 -d > build_redos/kdy.asc
  script:
    - docker run --rm -e KEY_FILE=kdy.asc -v $PWD/build_centos:/build docker-picodata.binary.picodata.io/rpmsign:centos7
    - docker run --rm -e KEY_FILE=kdy.asc -v $PWD/build_alt:/build docker-picodata.binary.picodata.io/rpmsign:centos7
    - docker run --rm -e KEY_FILE=kdy.asc -v $PWD/build_redos:/build docker-picodata.binary.picodata.io/rpmsign:centos7
  artifacts:
    paths:
      - build_centos/picodata*.rpm
      - build_alt/picodata*.rpm
      - build_redos/picodata*.rpm
  dependencies:
    - pack-centos
    - pack-altlinux
    - pack-redos

sign-astralinux-packages:
  variables:
    DOCKER_AUTH_CONFIG: $DOCKER_AUTH_RO
  stage: sign
  tags:
    - shell
  only:
    - web
    - tags
  before_script:
    - echo "$GPG_KEY_ASTRA" | base64 -d > build_astra/pico.asc
    - echo "$GPG_PASS_ASTRA" > build_astra/pico.pass
  script:
    - docker run --rm -e KEY_FILE=pico.asc -e PASS_FILE=pico.pass -e SIGNER="5A7D5C9D749260B6CCD24D72A45397D5554CBECD" -v $PWD/build_astra:/build docker-picodata.binary.picodata.io/astrasign:orel-2.12
  artifacts:
    paths:
      - build_astra/picodata*_signed.deb
  dependencies:
    - pack-astralinux

deploy-packages:
  stage: deploy
  tags:
    - shell
  only:
    - web
    - tags
  before_script:
    - eval $(ssh-agent -s)
    - echo "$DEPLOY_PROD_SSH_KEY" | base64 -d | ssh-add -
  script:
    # CentOS 7
    - echo "Deploying rpm-centos7-packet..."
    - scp -o stricthostkeychecking=no build_centos/picodata*.el7.*rpm ansible@94.26.239.246:/data/nginx/www/packrepo/tarantool-picodata/el/7/x86_64/
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "cd /data/nginx/www/packrepo/tarantool-picodata/el/7/ && createrepo --update x86_64 && gpg --no-tty --yes -u kdy@picodata.io --detach-sign --armor x86_64/repodata/repomd.xml"
    - echo "rpm-centos7-packet successfully deployed."
    - echo
    # CentOS 8
    - echo "Deploying rpm-centos8-packet..."
    - scp -o stricthostkeychecking=no build_centos/picodata*.el8.*rpm ansible@94.26.239.246:/data/nginx/www/packrepo/tarantool-picodata/el/8/x86_64/
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "cd /data/nginx/www/packrepo/tarantool-picodata/el/8/ && createrepo --update x86_64 && gpg --no-tty --yes -u kdy@picodata.io --detach-sign --armor x86_64/repodata/repomd.xml"
    - echo "rpm-centos8-packet successfully deployed."
    - echo
    # Ubuntu focal
    - echo "Deploying ubuntu focal deb-packets..."
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p ~/.deb/ubuntu"
    - scp -o stricthostkeychecking=no build_focal/picodata*deb ansible@94.26.239.246:.deb/ubuntu/
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/ubuntu/ -C main includedeb focal ~/.deb/ubuntu/picodata*focal*deb; rm ~/.deb/ubuntu/picodata*focal*deb"
    - echo "ubuntu focal deb-packets successfully deployed."
    - echo
    # Ubuntu jammy
    - echo "Deploying ubuntu jammy deb-packets..."
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p ~/.deb/ubuntu"
    - scp -o stricthostkeychecking=no build_jammy/picodata*deb ansible@94.26.239.246:.deb/ubuntu/
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/ubuntu/ -C main includedeb jammy ~/.deb/ubuntu/picodata*jammy*deb; rm ~/.deb/ubuntu/picodata*jammy*deb"
    - echo "ubuntu jammy deb-packets successfully deployed."
    - echo
    # Debian
    - echo "Deploying debian packets..."
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p ~/.deb/debian"
    - scp -o stricthostkeychecking=no build_debian/picodata*deb ansible@94.26.239.246:.deb/debian/
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/debian/ -C main includedeb bullseye ~/.deb/debian/picodata*bullseye*deb; rm ~/.deb/debian/picodata*bullseye*deb"
    - echo "debian packets successfully deployed."
    - echo
    # Altlinux p9
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p /tmp/altlinux/"
    - echo "Deploying altlinux-p9 packet..."
    - scp -o stricthostkeychecking=no build_alt/picodata*.p9.*rpm ansible@94.26.239.246:/tmp/altlinux/
    - echo "altlinux-p9 packet successfully deployed."
    - echo
    # Altlinux p10
    - echo "Deploying altlinux-p10 packet..."
    - scp -o stricthostkeychecking=no build_alt/picodata*.p10.*rpm ansible@94.26.239.246:/tmp/altlinux/
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "/usr/local/bin/repogen.sh"
    - echo "altlinux-p10 packet successfully deployed."
    - echo
    # RedOS
    - echo "Deploying RedOS 7 packet..."
    - scp -o stricthostkeychecking=no build_redos/picodata*.el7.*rpm ansible@94.26.239.246:/data/nginx/www/packrepo/tarantool-picodata/redos/7/x86_64/
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "cd /data/nginx/www/packrepo/tarantool-picodata/redos/7/ && createrepo --update x86_64 && gpg --no-tty --yes -u kdy@picodata.io --detach-sign --armor x86_64/repodata/repomd.xml"
    - echo "RedOS 7 packet successfully deployed."
    - echo
    # Astralinux
    - echo "Deploying Astralinux packet..."
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p ~/.deb/astra"
    - scp -o stricthostkeychecking=no build_astra/picodata*_signed.deb ansible@94.26.239.246:.deb/astra/
    - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/astra -C main includedeb orel ~/.deb/astra/picodata*_signed.deb; rm -rf ~/.deb/astra"
    - echo "Astralinux-packets successfully deployed."
    - echo
  dependencies:
    - pack-ubuntu
    - pack-debian
    - sign-rpm-packages
    - sign-astralinux-packages

deploy-docker:
  stage: deploy
  tags:
    - shell
  only:
    - web
    - tags
  variables:
    GIT_DEPTH: 100
    GIT_STRATEGY: fetch
    GIT_SUBMODULE_STRATEGY: recursive
  before_script:
    - export PATH=docker-build-base:$PATH
    - *fetch-tags
    - mkdir -p $CI_PROJECT_DIR/.docker
    - echo $DOCKER_AUTH_RW > $CI_PROJECT_DIR/.docker/config.json
  script:
    - |
      # Rebuild and push docker images
      for image in picodata picodata-diag; do
        ci-log-section start "deploy-docker-${image}" Building and pushing docker image ${image}
        docker build \
          --label GIT_COMMIT=${CI_COMMIT_SHA} \
          -t ${REGISTRY}/${image}:latest \
          -f helm/${image}.Dockerfile .
        docker --config $CI_PROJECT_DIR/.docker push ${REGISTRY}/${image}:latest
        ci-log-section end "deploy-docker-${image}"
      done

.check-deployment:
  stage: check-deployment
  tags:
    - docker
  only:
    - web
    - tags
  image: ${BASE_IMAGE}
  variables:
    DOCKER_AUTH_CONFIG: $DOCKER_AUTH_RO
  needs:
    - deploy-packages

check-deployment-rpm:
  extends: .check-deployment
  parallel:
    matrix:
      - BASE_IMAGE: centos:7
        PACKAGE: el/7/x86_64/picodata-release-1.1.1.0-1.el7.x86_64.rpm
        GIT_FETCH_PARAM: depth
      - BASE_IMAGE: rockylinux:8
        PACKAGE: el/8/x86_64/picodata-release-1.1.1.0-1.el8.x86_64.rpm
        GIT_FETCH_PARAM: deepen
      - BASE_IMAGE: packpack/packpack:redos-7.3
        PACKAGE: redos/7/x86_64/picodata-release-1.1.1.0-1.el7.x86_64.rpm
        GIT_FETCH_PARAM: deepen
  before_script:
    - yum install -y git
    - until git describe; do git fetch --${GIT_FETCH_PARAM} 100; done
    - export VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p')
  script:
    - rpm --import https://download.picodata.io/tarantool-picodata/el/RPM-GPG-KEY-kdy
    - yum install -y https://download.picodata.io/tarantool-picodata/${PACKAGE}
    - yum install -y picodata-${VER}

check-deployment-deb:
  extends: .check-deployment
  variables:
    DEBIAN_FRONTEND: noninteractive
    TZ: Europe/Moscow
  parallel:
    matrix:
      - BASE_IMAGE: debian:bullseye
      - BASE_IMAGE: ubuntu:focal
      - BASE_IMAGE: ubuntu:jammy
  before_script:
    - apt update
    - apt install -y curl gpg software-properties-common git
    - export DIST=$(lsb_release -si | tr [:upper:] [:lower:])
    - export CODENAME=$(lsb_release -sc)
    - until git describe; do git fetch --deepen 100; done
    - export VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p')
  script:
    - curl -s https://download.picodata.io/tarantool-picodata/ubuntu/picodata.gpg.key | gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/picodata.gpg --import
    - chmod 644 /etc/apt/trusted.gpg.d/picodata.gpg
    - add-apt-repository -y "deb [arch=amd64] https://download.picodata.io/tarantool-picodata/${DIST}/ ${CODENAME} main"
    - apt update
    - apt install -y picodata=${VER}-${CODENAME}

check-deployment-alt:
  extends: .check-deployment
  parallel:
    matrix:
      - DIST: p10
      - DIST: p9
  image: docker.binary.picodata.io/altlinux/base:${DIST}
  before_script:
    - apt-get update
    - apt-get install -y curl git apt-https
    - until git describe; do git fetch --deepen 100; done
    - export VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p')
  script:
    # Checking for absent dependencies
    - curl https://download.picodata.io/tarantool-picodata/altlinux/${DIST}/x86_64/RPMS.main/picodata-${VER}-1.${DIST}.x86_64.rpm -o picodata.rpm
    - apt-get install -y ./picodata.rpm
    - apt-get remove -y picodata
    # Checking for install from repo
    - apt-get install -y https://download.picodata.io/tarantool-picodata/altlinux/${DIST}/picodata-release-1.0.2.7-1.${DIST}.x86_64.rpm
    - apt-get update
    - apt-get install -y picodata=${VER}

check-deployment-astra:
  extends: .check-deployment
  image: docker-picodata.binary.picodata.io/astra/orel:2.12
  before_script:
    - apt-get update
    - apt-get install -y curl git apt-transport-https
    - until git describe; do git fetch --deepen 100; done
    - export VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p')
  script:
    - curl -s https://download.picodata.io/tarantool-picodata/ubuntu/picodata.gpg.key | gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/picodata.gpg --import
    - chmod 644 /etc/apt/trusted.gpg.d/picodata.gpg
    - echo "deb [arch=amd64] https://download.picodata.io/tarantool-picodata/astra/ orel main" > /etc/apt/sources.list.d/picodata.list
    - apt-get update
    - apt-get install -y picodata=${VER}-1