stages: - build-base-image - test - pack - sign - deploy - check-deployment workflow: # See https://docs.gitlab.com/ee/ci/jobs/job_control.html#avoid-duplicate-pipelines rules: # To avoid duplicate pipelines we disable merge request events, # leaving only pushes and manual triggering. - if: $CI_PIPELINE_SOURCE == "merge_request_event" when: never - if: $CI_PIPELINE_SOURCE == "push" - if: $CI_PIPELINE_SOURCE == "web" variables: REGISTRY: docker-public.binary.picodata.io BASE_IMAGE: ${REGISTRY}/picodata-build-base CARGO_HOME: /shared-storage/picodata/.cargo CACHE_PATHS: target .venv picodata-webui/node_modules CACHE_ARCHIVE: /shared-storage/picodata/cache.tar # job:rules explained: # # - if build-base changes on master branch (compared to HEAD~1) # * build-base-image (with tag latest) and push # * test (on base-image:latest) # - if build-base changes on development branch (compared to master) # * build-base-image (with tag sha) # * test (on base-image:sha) # - else (if build-base doesn't change) # * skip build-base-image # * just test (on base-image:latest) # # Anchor syntax explained here: # https://docs.gitlab.com/ee/ci/yaml/yaml_optimization.html # .rules: - &if-build-base-changes-on-master-branch if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH changes: # implies compare_to HEAD~1 paths: &build-base-changes-paths - docker-build-base/** - .gitlab-ci.yml - &if-build-base-changes-on-dev-branch if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH changes: compare_to: master paths: *build-base-changes-paths - &else {} build-base-image: stage: build-base-image tags: - shell-yc rules: - <<: *if-build-base-changes-on-master-branch variables: BASE_IMAGE_TAG: latest - <<: *if-build-base-changes-on-dev-branch variables: BASE_IMAGE_TAG: ${CI_COMMIT_SHA} - <<: *else when: never variables: GIT_DEPTH: 1 GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: none script: - docker pull ${BASE_IMAGE}:latest || true - > docker build --cache-from ${BASE_IMAGE}:latest --label GIT_COMMIT=${CI_COMMIT_SHA} -t ${BASE_IMAGE}:${BASE_IMAGE_TAG} -f ./docker-build-base/Dockerfile ./docker-build-base - | # Push image to registry if [ "${CI_COMMIT_BRANCH}" == "${CI_DEFAULT_BRANCH}" ]; then echo "Pushing ${BASE_IMAGE}:${BASE_IMAGE_TAG}" mkdir -p $CI_PROJECT_DIR/.docker echo $DOCKER_AUTH_RW > $CI_PROJECT_DIR/.docker/config.json docker --config $CI_PROJECT_DIR/.docker/ push ${BASE_IMAGE}:${BASE_IMAGE_TAG} else echo "Skip pushing image on a non-master branch" fi test: stage: test tags: - docker-yc rules: - <<: *if-build-base-changes-on-master-branch variables: BASE_IMAGE_TAG: latest - <<: *if-build-base-changes-on-dev-branch variables: BASE_IMAGE_TAG: ${CI_COMMIT_SHA} - <<: *else variables: BASE_IMAGE_TAG: latest image: name: ${BASE_IMAGE}:${BASE_IMAGE_TAG} pull_policy: if-not-present variables: GIT_DEPTH: 100 GIT_SUBMODULE_STRATEGY: recursive RUST_BACKTRACE: full before_script: # Gitlab CI implicitly clones specific refs (e.g. `refs/pipelines/xxxxxxx`), # but it doesn't imply fetching tags. We clone them manually with the # `git fetch` command. # # Tags in `tarantool-sys` and `luajit` submodules are necessary for # the build scripts. Without them the job fails. - &fetch-tags | # Fetch tags ci-log-section start "fetch-submodule-tags" Fetching tags for submodules for s in tarantool-sys tarantool-sys/third_party/luajit; do echo "Fetching tag for $s" pushd $s until git describe; do git fetch --deepen 100; done popd done ci-log-section end "fetch-submodule-tags" # Gitlab CI caching is shit. So we implement it manually - | # Restore cache if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then echo "Skip restoring cache on the master branch" elif [ -f "${CACHE_ARCHIVE}" ]; then ci-log-section start "restore-cache" Restoring cache from ${CACHE_ARCHIVE} ... tar -xf ${CACHE_ARCHIVE} echo "Ok" du -sh ${CACHE_PATHS} || true ci-log-section end "restore-cache" else echo "No cache found" fi script: - cargo -V - cargo build --locked - cargo build --features webui --locked # There are no Rust tests for `webui` feature. # It will be checked during integration tests. - cargo test --locked - cargo fmt -- -v --check - cargo clippy --version - cargo clippy --features webui -- --deny clippy::all - | # Pipenv install ci-log-section start "pipenv-install" Installing pip dependencies ... PIPENV_VENV_IN_PROJECT=1 PIP_NO_CACHE_DIR=true python3.10 -m pipenv install --deploy ci-log-section end "pipenv-install" - pipenv run pytest --numprocesses auto -v - pipenv run lint - | # Save cache if [ "$CI_COMMIT_BRANCH" == "$CI_DEFAULT_BRANCH" ]; then ci-log-section start "save-cache" Saving cache to ${CACHE_ARCHIVE} ... du -sh ${CACHE_PATHS} || true TMPEXT=$RANDOM tar -cf "${CACHE_ARCHIVE}.${TMPEXT}" ${CACHE_PATHS} mv -f "${CACHE_ARCHIVE}.${TMPEXT}" "${CACHE_ARCHIVE}" echo Ok du -sh ${CACHE_ARCHIVE} ci-log-section end "save-cache" else echo "Skip saving cache on a non-master branch" fi test-docker: stage: test tags: - shell rules: - if: $CI_COMMIT_BRANCH != $CI_DEFAULT_BRANCH changes: compare_to: master paths: - helm/picodata.Dockerfile - helm/picodata-diag.Dockerfile - if: $CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH when: on_success - # else when: never variables: GIT_DEPTH: 100 GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: recursive before_script: - export PATH=docker-build-base:$PATH - *fetch-tags script: - | # Build docker images for image in picodata picodata-diag; do ci-log-section start "test-docker-${image}" Building docker image ${image} docker build \ --label GIT_COMMIT=${CI_COMMIT_SHA} \ -f helm/${image}.Dockerfile . ci-log-section end "test-docker-${image}" done .pack: tags: - shell-limit-1 only: - web - tags variables: PRESERVE_ENVVARS: VER_TNT GIT_DEPTH: 100 GIT_SUBMODULE_STRATEGY: recursive before_script: - git clone https://github.com/packpack/packpack.git packpack - | # Describe tarantool-sys pushd tarantool-sys until git describe; do git fetch --deepen 100; done export VER_TNT=$( git describe --long | # 2.10.5-17-gf818b9108 -> '2.10.5.17' # 2.11.0-entrypoint-1137-g4686b909 -> '2.11.0.1137' # 3.0.0-alpha1-14-g342c242-dev -> '3.0.0.14' sed -n 's/-[a-z]\+[a-z0-9]*//p' |sed -n 's/^\([0-9\.]*\)-\([0-9]*\)\(-\([a-z0-9]*\)\)*/\1.\2/p' ); echo $VER_TNT [ "$VER_TNT" == "" ] && false popd - | # Describe picodata until git describe; do git fetch --deepen 100; done git describe --long timeout: 2h dependencies: [] pack-centos: stage: pack extends: .pack script: - sed -i "s/(id -u)/(id -u) -o/g" packpack/packpack - OS=centos DIST=7 BUILDDIR=$PWD/build_centos packpack/packpack - OS=centos DIST=8 BUILDDIR=$PWD/build_centos packpack/packpack artifacts: paths: - build_centos/picodata*.rpm pack-ubuntu: stage: pack extends: .pack script: - OS=ubuntu DIST=focal BUILDDIR=$PWD/build_${DIST}/ RELEASE=${DIST} packpack/packpack - OS=ubuntu DIST=jammy BUILDDIR=$PWD/build_${DIST}/ RELEASE=${DIST} packpack/packpack artifacts: paths: - build_focal/*.deb - build_jammy/*.deb pack-debian: stage: pack extends: .pack script: - OS=debian DIST=bullseye BUILDDIR=$PWD/build_debian/ RELEASE=${DIST} packpack/packpack artifacts: paths: - build_debian/*.deb pack-altlinux: stage: pack extends: .pack script: - DOCKER_REPO=docker-picodata.binary.picodata.io/packpack/alt DOCKER_IMAGE=p10 BUILDDIR=$PWD/build_alt packpack/packpack - DOCKER_REPO=docker-picodata.binary.picodata.io/packpack/alt DOCKER_IMAGE=p9 BUILDDIR=$PWD/build_alt packpack/packpack artifacts: paths: - build_alt/picodata*.rpm pack-redos: stage: pack extends: .pack script: - OS=redos DIST=7.3 BUILDDIR=$PWD/build_redos/ packpack/packpack artifacts: paths: - build_redos/picodata*.rpm pack-astralinux: stage: pack extends: .pack script: - DOCKER_REPO=docker-picodata.binary.picodata.io/packpack/astra DOCKER_IMAGE=orel-2.12 BUILDDIR=$PWD/build_astra packpack/packpack artifacts: paths: - build_astra/*.deb pack-macos-arm: stage: pack extends: .pack tags: - mac-dev-m1 variables: PKGNAME: 'picodata' BUILDDIR: 'build_mac_arm' MACOS: 'ventura' script: | export CARGO_HOME=$HOME/.cargo VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p') MAJOR=$(echo $VER | cut -f 1-2 -d '.') make reset-submodules echo "-------------------------------------------------------------------" echo "Prepare bottle" echo "-------------------------------------------------------------------" make build export DESTDIR="$BUILDDIR/$PKGNAME@$MAJOR/$VER" make install cp AUTHORS README.md LICENSE $DESTDIR mv $DESTDIR/usr/bin $DESTDIR/ && rm -rf $DESTDIR/usr mkdir -p $DESTDIR/.bottle/{etc,var} pushd $BUILDDIR tar -caf $PKGNAME@$MAJOR-$VER.arm64_$MACOS.bottle.tar.gz $PKGNAME@$MAJOR/$VER popd echo "-------------------------------------------------------------------" echo "Prepare tarball with sources" echo "-------------------------------------------------------------------" mkdir $BUILDDIR/$PKGNAME-$VER rsync -r -l \ --exclude=.git --exclude='.gitignore' --exclude='.gitmodules' \ --exclude=FreeBSD --exclude=debian --exclude=rpm --exclude=rump \ --exclude=apk --exclude=$BUILDDIR --exclude=packpack \ * $BUILDDIR/$PKGNAME-$VER/ cd $BUILDDIR tar --uid=0 --gid=0 -caPf $PKGNAME-$VER.tar.gz $PKGNAME-$VER shasum -a 256 $PKGNAME*$VER*.tar.gz artifacts: paths: - $BUILDDIR/*.tar.gz sign-rpm-packages: variables: DOCKER_AUTH_CONFIG: $DOCKER_AUTH_RO stage: sign tags: - shell only: - web - tags before_script: - echo "$GPG_KEY_KDY" | base64 -d > build_centos/kdy.asc - echo "$GPG_KEY_KDY" | base64 -d > build_alt/kdy.asc - echo "$GPG_KEY_KDY" | base64 -d > build_redos/kdy.asc script: - docker run --rm -e KEY_FILE=kdy.asc -v $PWD/build_centos:/build docker-picodata.binary.picodata.io/rpmsign:centos7 - docker run --rm -e KEY_FILE=kdy.asc -v $PWD/build_alt:/build docker-picodata.binary.picodata.io/rpmsign:centos7 - docker run --rm -e KEY_FILE=kdy.asc -v $PWD/build_redos:/build docker-picodata.binary.picodata.io/rpmsign:centos7 artifacts: paths: - build_centos/picodata*.rpm - build_alt/picodata*.rpm - build_redos/picodata*.rpm dependencies: - pack-centos - pack-altlinux - pack-redos sign-astralinux-packages: variables: DOCKER_AUTH_CONFIG: $DOCKER_AUTH_RO stage: sign tags: - shell only: - web - tags before_script: - echo "$GPG_KEY_ASTRA" | base64 -d > build_astra/pico.asc - echo "$GPG_PASS_ASTRA" > build_astra/pico.pass script: - docker run --rm -e KEY_FILE=pico.asc -e PASS_FILE=pico.pass -e SIGNER="5A7D5C9D749260B6CCD24D72A45397D5554CBECD" -v $PWD/build_astra:/build docker-picodata.binary.picodata.io/astrasign:orel-2.12 artifacts: paths: - build_astra/picodata*_signed.deb dependencies: - pack-astralinux deploy-packages: stage: deploy tags: - shell only: - web - tags when: manual before_script: - eval $(ssh-agent -s) - echo "$DEPLOY_PROD_SSH_KEY" | base64 -d | ssh-add - script: # CentOS 7 - echo "Deploying rpm-centos7-packet..." - scp -o stricthostkeychecking=no build_centos/picodata*.el7.*rpm ansible@94.26.239.246:/data/nginx/www/packrepo/tarantool-picodata/el/7/x86_64/ - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "cd /data/nginx/www/packrepo/tarantool-picodata/el/7/ && createrepo --update x86_64 && gpg --no-tty --yes -u kdy@picodata.io --detach-sign --armor x86_64/repodata/repomd.xml" - echo "rpm-centos7-packet successfully deployed." - echo # CentOS 8 - echo "Deploying rpm-centos8-packet..." - scp -o stricthostkeychecking=no build_centos/picodata*.el8.*rpm ansible@94.26.239.246:/data/nginx/www/packrepo/tarantool-picodata/el/8/x86_64/ - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "cd /data/nginx/www/packrepo/tarantool-picodata/el/8/ && createrepo --update x86_64 && gpg --no-tty --yes -u kdy@picodata.io --detach-sign --armor x86_64/repodata/repomd.xml" - echo "rpm-centos8-packet successfully deployed." - echo # Ubuntu focal - echo "Deploying ubuntu focal deb-packets..." - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p ~/.deb/ubuntu" - scp -o stricthostkeychecking=no build_focal/picodata*deb ansible@94.26.239.246:.deb/ubuntu/ - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/ubuntu/ -C main includedeb focal ~/.deb/ubuntu/picodata*focal*deb; rm ~/.deb/ubuntu/picodata*focal*deb" - echo "ubuntu focal deb-packets successfully deployed." - echo # Ubuntu jammy - echo "Deploying ubuntu jammy deb-packets..." - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p ~/.deb/ubuntu" - scp -o stricthostkeychecking=no build_jammy/picodata*deb ansible@94.26.239.246:.deb/ubuntu/ - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/ubuntu/ -C main includedeb jammy ~/.deb/ubuntu/picodata*jammy*deb; rm ~/.deb/ubuntu/picodata*jammy*deb" - echo "ubuntu jammy deb-packets successfully deployed." - echo # Debian - echo "Deploying debian packets..." - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p ~/.deb/debian" - scp -o stricthostkeychecking=no build_debian/picodata*deb ansible@94.26.239.246:.deb/debian/ - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/debian/ -C main includedeb bullseye ~/.deb/debian/picodata*bullseye*deb; rm ~/.deb/debian/picodata*bullseye*deb" - echo "debian packets successfully deployed." - echo # Altlinux p9 - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p /tmp/altlinux/" - echo "Deploying altlinux-p9 packet..." - scp -o stricthostkeychecking=no build_alt/picodata*.p9.*rpm ansible@94.26.239.246:/tmp/altlinux/ - echo "altlinux-p9 packet successfully deployed." - echo # Altlinux p10 - echo "Deploying altlinux-p10 packet..." - scp -o stricthostkeychecking=no build_alt/picodata*.p10.*rpm ansible@94.26.239.246:/tmp/altlinux/ - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "/usr/local/bin/repogen.sh" - echo "altlinux-p10 packet successfully deployed." - echo # RedOS - echo "Deploying RedOS 7 packet..." - scp -o stricthostkeychecking=no build_redos/picodata*.el7.*rpm ansible@94.26.239.246:/data/nginx/www/packrepo/tarantool-picodata/redos/7/x86_64/ - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "cd /data/nginx/www/packrepo/tarantool-picodata/redos/7/ && createrepo --update x86_64 && gpg --no-tty --yes -u kdy@picodata.io --detach-sign --armor x86_64/repodata/repomd.xml" - echo "RedOS 7 packet successfully deployed." - echo # Astralinux - echo "Deploying Astralinux packet..." - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "mkdir -p ~/.deb/astra" - scp -o stricthostkeychecking=no build_astra/picodata*_signed.deb ansible@94.26.239.246:.deb/astra/ - ssh -o stricthostkeychecking=no ansible@94.26.239.246 "reprepro -b /data/nginx/www/packrepo/tarantool-picodata/astra -C main includedeb orel ~/.deb/astra/picodata*_signed.deb; rm -rf ~/.deb/astra" - echo "Astralinux-packets successfully deployed." - echo dependencies: - pack-ubuntu - pack-debian - sign-rpm-packages - sign-astralinux-packages deploy-macos-arm: stage: deploy tags: - mac-dev-m1 only: - web - tags when: manual variables: PKGNAME: 'picodata' BUILDDIR: 'build_mac_arm' script: | VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p') MAJOR=$(echo $VER | cut -f 1-2 -d '.') curl -v -H "Authorization: Basic $RAW_AUTH_RW" --upload-file $BUILDDIR/$PKGNAME-$VER.tar.gz $RAW_REGISTRY/brew/packages/ curl -v -H "Authorization: Basic $RAW_AUTH_RW" --upload-file $BUILDDIR/$PKGNAME@$MAJOR-$VER.*.bottle.tar.gz $RAW_REGISTRY/brew/bottle/ dependencies: - pack-macos-arm deploy-docker: stage: deploy tags: - shell only: - web - tags variables: GIT_DEPTH: 100 GIT_STRATEGY: fetch GIT_SUBMODULE_STRATEGY: recursive before_script: - export PATH=docker-build-base:$PATH - *fetch-tags - mkdir -p $CI_PROJECT_DIR/.docker - echo $DOCKER_AUTH_RW > $CI_PROJECT_DIR/.docker/config.json script: - | # Rebuild and push docker images for image in picodata picodata-diag; do ci-log-section start "deploy-docker-${image}" Building and pushing docker image ${image} docker build \ --label GIT_COMMIT=${CI_COMMIT_SHA} \ -t ${REGISTRY}/${image}:latest \ -f helm/${image}.Dockerfile . docker --config $CI_PROJECT_DIR/.docker push ${REGISTRY}/${image}:latest ci-log-section end "deploy-docker-${image}" done .check-deployment: stage: check-deployment tags: - docker only: - web - tags image: ${BASE_IMAGE} variables: DOCKER_AUTH_CONFIG: $DOCKER_AUTH_RO needs: - deploy-packages check-deployment-rpm: extends: .check-deployment parallel: matrix: - BASE_IMAGE: centos:7 PACKAGE: el/7/x86_64/picodata-release-1.1.1.0-1.el7.x86_64.rpm GIT_FETCH_PARAM: depth - BASE_IMAGE: rockylinux:8 PACKAGE: el/8/x86_64/picodata-release-1.1.1.0-1.el8.x86_64.rpm GIT_FETCH_PARAM: deepen - BASE_IMAGE: packpack/packpack:redos-7.3 PACKAGE: redos/7/x86_64/picodata-release-1.1.1.0-1.el7.x86_64.rpm GIT_FETCH_PARAM: deepen before_script: - yum install -y git - until git describe; do git fetch --${GIT_FETCH_PARAM} 100; done - export VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p') script: - rpm --import https://download.picodata.io/tarantool-picodata/el/RPM-GPG-KEY-kdy - yum install -y https://download.picodata.io/tarantool-picodata/${PACKAGE} - yum install -y picodata-${VER} check-deployment-deb: extends: .check-deployment variables: DEBIAN_FRONTEND: noninteractive TZ: Europe/Moscow parallel: matrix: - BASE_IMAGE: debian:bullseye - BASE_IMAGE: ubuntu:focal - BASE_IMAGE: ubuntu:jammy before_script: - apt update - apt install -y curl gpg software-properties-common git - export DIST=$(lsb_release -si | tr [:upper:] [:lower:]) - export CODENAME=$(lsb_release -sc) - until git describe; do git fetch --deepen 100; done - export VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p') script: - curl -s https://download.picodata.io/tarantool-picodata/ubuntu/picodata.gpg.key | gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/picodata.gpg --import - chmod 644 /etc/apt/trusted.gpg.d/picodata.gpg - add-apt-repository -y "deb [arch=amd64] https://download.picodata.io/tarantool-picodata/${DIST}/ ${CODENAME} main" - apt update - apt install -y picodata=${VER}-${CODENAME} check-deployment-alt: extends: .check-deployment parallel: matrix: - DIST: p10 - DIST: p9 image: docker.binary.picodata.io/altlinux/base:${DIST} before_script: - apt-get update - apt-get install -y curl git apt-https - until git describe; do git fetch --deepen 100; done - export VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p') script: # Checking for absent dependencies - curl https://download.picodata.io/tarantool-picodata/altlinux/${DIST}/x86_64/RPMS.main/picodata-${VER}-1.${DIST}.x86_64.rpm -o picodata.rpm - apt-get install -y ./picodata.rpm - apt-get remove -y picodata # Checking for install from repo - apt-get install -y https://download.picodata.io/tarantool-picodata/altlinux/${DIST}/picodata-release-1.0.2.7-1.${DIST}.x86_64.rpm - apt-get update - apt-get install -y picodata=${VER} check-deployment-astra: extends: .check-deployment image: docker-picodata.binary.picodata.io/astra/orel:2.12 before_script: - apt-get update - apt-get install -y curl git apt-transport-https - until git describe; do git fetch --deepen 100; done - export VER=$(git describe --long | sed -n 's/^\([0-9\.]*\)-\([0-9]*\)-\([a-z0-9]*\)/\1.\2/p') script: - curl -s https://download.picodata.io/tarantool-picodata/ubuntu/picodata.gpg.key | gpg --no-default-keyring --keyring gnupg-ring:/etc/apt/trusted.gpg.d/picodata.gpg --import - chmod 644 /etc/apt/trusted.gpg.d/picodata.gpg - echo "deb [arch=amd64] https://download.picodata.io/tarantool-picodata/astra/ orel main" > /etc/apt/sources.list.d/picodata.list - apt-get update - apt-get install -y picodata=${VER}-1