From 933dfd09116699d65316c494cf534fb8399a80c5 Mon Sep 17 00:00:00 2001
From: Vitaly Shunkov <v.shunkov@picodata.io>
Date: Mon, 22 Jul 2024 17:46:48 +0300
Subject: [PATCH] ci: retry tag fetches, refactor

---
 .gitlab-ci.yml    | 21 +++++---------------
 tools/get_tags.py | 50 +++++++++++++++++++++++++++++++++++++++++++++++
 2 files changed, 55 insertions(+), 16 deletions(-)
 create mode 100755 tools/get_tags.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 592b9d8ef0..2ca4e792e8 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -36,6 +36,7 @@ variables:
   KANIKO_REGISTRY_MIRROR: docker-proxy.binary.picodata.io
   FF_USE_FASTZIP: "true"
   CACHE_COMPRESSION_LEVEL: "fastest"
+  GIT_DEPTH: 1
   GET_SOURCES_ATTEMPTS: 3
   PARENT_BRANCH: $CI_COMMIT_BRANCH
   PARENT_CI_COMMIT_SHA: $CI_COMMIT_SHA
@@ -122,7 +123,7 @@ build-base-image:
   interruptible: true
   stage: test
   variables:
-    GIT_DEPTH: 100
+    GIT_STRATEGY: clone
     GIT_SUBMODULE_STRATEGY: recursive
     RUST_BACKTRACE: full
   before_script:
@@ -133,15 +134,10 @@ build-base-image:
     # Tags in `tarantool-sys` and `luajit` submodules are necessary for
     # the build scripts. Without them the job fails.
     - &fetch-tags |
-      # Fetch tags
       ci-log-section start "fetch-submodule-tags" Fetching tags for submodules
-      for s in . tarantool-sys tarantool-sys/third_party/luajit; do
-        echo "Fetching tag for $s"
-        pushd $s
-        until git describe; do git fetch --deepen 100; done
-        popd
-      done
-      ci-log-section end "fetch-submodule-tags"
+      ./tools/get_tags.py . tarantool-sys tarantool-sys/third_party/luajit      
+      ci-log-section end "fetch-submodule-tags"    
+
 
 .parallel:
   parallel:
@@ -172,8 +168,6 @@ test-linux:
   image:
     name: ${BASE_IMAGE}:${BASE_IMAGE_TAG}
     pull_policy: if-not-present
-  variables:
-    GIT_DEPTH: 1
   cache:
     - <<: *py_cache
     - <<: *base_cache
@@ -246,7 +240,6 @@ lint:
     name: ${BASE_IMAGE}:${BASE_IMAGE_TAG}
     pull_policy: if-not-present
   variables:
-    GIT_DEPTH: 1
     GIT_SUBMODULE_STRATEGY: recursive
   cache:
     - <<: *py_cache
@@ -294,7 +287,6 @@ test-patch-picodata:
   extends: .test
   <<: *test-patch-rules
   variables:
-    GIT_DEPTH: 1
     GIT_SUBMODULE_STRATEGY: recursive
   image:
     name: ${BASE_IMAGE}:${BASE_IMAGE_LATEST}
@@ -324,7 +316,6 @@ test-patch-tarantool:
   extends: .test
   <<: *test-patch-rules
   variables:
-    GIT_DEPTH: 1
     GIT_SUBMODULE_STRATEGY: recursive
   image:
     name: ${BASE_IMAGE}:${BASE_IMAGE_LATEST}
@@ -508,14 +499,12 @@ deploy-docker:
     name: ${BASE_IMAGE}:${BASE_IMAGE_LATEST}
   variables:
     VER: $CI_COMMIT_SHORT_SHA
-    GIT_DEPTH: 100
     GIT_SUBMODULE_STRATEGY: recursive
   before_script:
     # To conduct stress tests, we require the most recent picodata tag. However,
     # creating a shallow copy with $GIT_DEPTH commits might not include this tag.
     # Fetching all commits takes too much time, so we incrementally download more
     # chunks of commits until we find the required tag.
-    - until git describe; do git fetch --deepen 100; done
     - *fetch-tags
   script:
     - cargo build --locked --release --features webui
diff --git a/tools/get_tags.py b/tools/get_tags.py
new file mode 100755
index 0000000000..26bd4a5377
--- /dev/null
+++ b/tools/get_tags.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import argparse
+import subprocess
+import pathlib
+import os
+from time import sleep
+
+
+GET_SOURCES_ATTEMPTS = int(os.environ.get('GET_SOURCES_ATTEMPTS', 3))
+PROJECT_DIR = pathlib.Path(__file__).parent.parent
+
+
+def run_shell(path, shell=True, executable='/bin/bash', text=True):
+    retry = GET_SOURCES_ATTEMPTS
+    timeout = 3
+    while retry > 0:
+        try:
+            while True:
+                result = ""
+                proc = subprocess.run("git describe",
+                                      shell=shell, executable=executable, text=text,
+                                      cwd="{}/{}".format(PROJECT_DIR, path))
+                result = proc.stdout
+                code = proc.returncode
+                if not code:
+                    return
+
+                print("fetching tag for", path)
+                subprocess.run(
+                    "git fetch --deepen 50",
+                    shell=shell,
+                    executable=executable,
+                    text=text,
+                    cwd="{}/{}".format(PROJECT_DIR, path),
+                )
+        except Exception as e:
+            print("can't run: " + str(e))
+            retry -= 1
+            sleep(timeout)
+    return result
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(prog="GetGitTags", description="Get project tags")
+    parser.add_argument("dirs", nargs="*", default=".", type=str)
+    args = parser.parse_args()
+
+    for path in args.dirs:
+        run_shell(path)
-- 
GitLab