Last active 1 month ago

Получить все исходники, доступные в Gitlab

botanikanet revised this gist 1 month ago. Go to revision

1 file changed, 183 insertions, 73 deletions

get_all_src.sh

@@ -1,106 +1,216 @@
1 1 #!/bin/bash
2 - set -ex
2 + set -euo pipefail
3 3
4 + # ── Colors ───────────────────────────────────────────────────────────
5 + RED='\033[0;31m'
6 + GREEN='\033[0;32m'
7 + YELLOW='\033[1;33m'
8 + CYAN='\033[0;36m'
9 + NC='\033[0m'
10 +
11 + # ── Logging ──────────────────────────────────────────────────────────
12 + log_info() { printf "${CYAN}[INFO]${NC} %s\n" "$*"; }
13 + log_warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$*"; }
14 + log_ok() { printf "${GREEN}[ OK ]${NC} %s\n" "$*"; }
15 + log_error() { printf "${RED}[ERR]${NC} %s\n" "$*" >&2; }
16 +
17 + # ── Usage ────────────────────────────────────────────────────────────
4 18 usage() {
5 - echo "basename ${0} get all src from Gitlab"
6 - echo "--help | -h Print this help"
7 - echo "--token-file= | -t <file> Set file with Gitlab token"
8 - echo "--gitlab-url=* | -g <url> Set Gtilab URL"
19 + cat <<EOF
20 + Usage: $(basename "$0") [OPTIONS]
21 +
22 + Clone every project from every group (and subgroups, recursively) on a
23 + GitLab instance, preserving the path_with_namespace directory structure.
24 +
25 + Options:
26 + -h, --help Show this help
27 + -t, --token-file=FILE File containing the GitLab personal access token
28 + -g, --gitlab-url=URL GitLab instance URL (default: https://git.mws-team.ru)
29 +
30 + Examples:
31 + $(basename "$0") -t ~/.gitlab_token
32 + $(basename "$0") --token-file=token.txt --gitlab-url=https://gitlab.com
33 + EOF
9 34 exit 1
10 35 }
11 36
12 - # Map long options to short ones and rebuild argv
37 + # ── Dependency check ─────────────────────────────────────────────────
38 + for cmd in curl jq git; do
39 + if ! command -v "$cmd" &>/dev/null; then
40 + log_error "Required command not found: ${cmd}"
41 + exit 1
42 + fi
43 + done
44 +
45 + # ── Parse arguments ──────────────────────────────────────────────────
46 + GITLAB_URL="https://git.mws-team.ru"
47 + TOKEN_FILE=""
48 +
13 49 translated=()
14 50 while (( $# )); do
15 51 case "$1" in
16 52 --help) translated+=("-h"); shift ;;
17 53 --token-file=*) translated+=("-t" "${1#*=}"); shift ;;
18 54 --gitlab-url=*) translated+=("-g" "${1#*=}"); shift ;;
19 - --) translated+=("--"); shift; translated+=("$@"); set -- ;;
55 + --) translated+=("--"); shift; translated+=("$@"); break ;;
20 56 *) translated+=("$1"); shift ;;
21 - esac
57 + esac
22 58 done
23 59 set -- "${translated[@]}"
24 60
25 - declare GITLAB_URL="https://git.mws-team.ru"
26 - declare TOKEN_FILE=""
27 -
28 61 while getopts ":ht:g:" opt; do
29 62 case "$opt" in
30 - h) usage ;;
31 - t) TOKEN_FILE=$OPTARG ;;
32 - g) GITLAB_URL=$OPTARG ;;
33 - :) echo "Missing arg for -$OPTARG"; usage ;;
34 - \?) echo "Invalid option: -$OPTARG"; usage ;;
63 + h) usage ;;
64 + t) TOKEN_FILE="$OPTARG" ;;
65 + g) GITLAB_URL="$OPTARG" ;;
66 + :) log_error "Missing argument for -${OPTARG}"; usage ;;
67 + \?) log_error "Invalid option: -${OPTARG}"; usage ;;
35 68 esac
36 69 done
70 + shift $((OPTIND - 1))
37 71
38 - function checks {
39 - if [ -z "${TOKEN_FILE}" ]
40 - then
41 - echo "Missing token file"
42 - usage
43 - fi
44 -
45 - if [ -z "${GITLAB_URL}" ]
46 - then
47 - echo "Missing gitlab url"
48 - usage
49 - fi
50 -
51 - if [ ! -f "${TOKEN_FILE}" ]
52 - then
53 - echo "Token file not found"
54 - usage
55 - fi
72 + # ── Validate inputs ─────────────────────────────────────────────────
73 + [[ -z "$TOKEN_FILE" ]] && { log_error "Missing --token-file"; usage; }
74 + [[ -z "$GITLAB_URL" ]] && { log_error "Missing --gitlab-url"; usage; }
75 + [[ ! -f "$TOKEN_FILE" ]] && { log_error "Token file not found: ${TOKEN_FILE}"; exit 1; }
76 +
77 + GITLAB_URL="${GITLAB_URL%/}"
78 +
79 + TOKEN="$(<"$TOKEN_FILE")"
80 + readonly TOKEN GITLAB_URL
81 +
82 + # ── GitLab API helpers ───────────────────────────────────────────────
83 +
84 + # Authenticated GET request; returns JSON body.
85 + gitlab_get() {
86 + curl --silent --fail --header "Authorization: Bearer ${TOKEN}" "$1"
56 87 }
57 88
58 - function get_subgroups {
59 - for PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups/${1}/subgroups?per_page=100 2>/dev/null| jq -r '."x-total-pages"[0]'))
60 - do
61 - curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups/${1}/subgroups?per_page=100&page=${PAGE}" | jq -r '.[].id'
89 + # Return total page count for a paginated endpoint.
90 + gitlab_total_pages() {
91 + local headers
92 + headers="$(curl -X HEAD --silent -w '%{header_json}' \
93 + --header "Authorization: Bearer ${TOKEN}" \
94 + -o /dev/null "$1" 2>/dev/null)" || true
95 +
96 + local pages
97 + pages="$(echo "$headers" | jq -r '."x-total-pages"[0] // "1"')"
98 + [[ -z "$pages" || "$pages" == "null" ]] && pages=1
99 + echo "$pages"
100 + }
101 +
102 + # Iterate all pages of an endpoint, call a callback with each extracted value.
103 + # Usage: gitlab_paginate "URL?per_page=100" '.[].id' callback_fn
104 + gitlab_paginate() {
105 + local base_url="$1"
106 + local jq_filter="$2"
107 + local callback="$3"
108 +
109 + local total_pages
110 + total_pages="$(gitlab_total_pages "$base_url")"
111 +
112 + local page
113 + for (( page = 1; page <= total_pages; page++ )); do
114 + local response
115 + response="$(gitlab_get "${base_url}&page=${page}")" || {
116 + log_warn "API request failed: ${base_url}&page=${page}"
117 + continue
118 + }
119 +
120 + local item
121 + while IFS= read -r item; do
122 + [[ -z "$item" ]] && continue
123 + "$callback" "$item"
124 + done < <(echo "$response" | jq -r "$jq_filter")
62 125 done
63 126 }
64 127
65 - function clone_project {
66 - PROJECT_INFO=$(curl --silent --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/projects/${1} | jq .)
67 - echo "INFO: Check project $(echo ${PROJECT_INFO} | jq -r '.path_with_namespace')"
68 - if [ ! -d "$(echo ${PROJECT_INFO} | jq -r '.path_with_namespace')" ]
69 - then
70 - echo "INFO: Clone project $(echo ${PROJECT_INFO} | jq -r '.path_with_namespace')"
71 - mkdir -p $(echo ${PROJECT_INFO} | jq -r '.path_with_namespace')
72 - git clone $(echo ${PROJECT_INFO} | jq -r '. | "\(.ssh_url_to_repo) \(.path_with_namespace)"')
128 + # ── Clone a single project ──────────────────────────────────────────
129 + CLONE_COUNT=0
130 + SKIP_COUNT=0
131 +
132 + clone_project() {
133 + local project_id="$1"
134 +
135 + local project_json
136 + project_json="$(gitlab_get "${GITLAB_URL}/api/v4/projects/${project_id}")" || {
137 + log_warn "Failed to fetch project ${project_id}"
138 + return
139 + }
140 +
141 + local path_ns ssh_url
142 + path_ns="$(echo "$project_json" | jq -r '.path_with_namespace')"
143 + ssh_url="$(echo "$project_json" | jq -r '.ssh_url_to_repo')"
144 +
145 + if [[ -z "$path_ns" || "$path_ns" == "null" ]]; then
146 + log_warn "Could not determine path for project ${project_id} — skipping"
147 + return
148 + fi
149 +
150 + if [[ -d "$path_ns" ]]; then
151 + log_info "Already cloned: ${path_ns}"
152 + (( SKIP_COUNT++ )) || true
153 + return
154 + fi
155 +
156 + log_info "Cloning: ${path_ns}"
157 + mkdir -p "$(dirname "$path_ns")"
158 + if git clone --quiet "$ssh_url" "$path_ns"; then
159 + log_ok "Cloned: ${path_ns}"
160 + (( CLONE_COUNT++ )) || true
161 + else
162 + log_warn "git clone failed for ${path_ns}"
73 163 fi
74 164 }
75 165
76 - checks
166 + # ── Clone all projects in a group ────────────────────────────────────
167 + clone_group_projects() {
168 + local group_id="$1"
169 + gitlab_paginate \
170 + "${GITLAB_URL}/api/v4/groups/${group_id}/projects?per_page=100&archived=no" \
171 + '.[].id' clone_project
172 + }
77 173
78 - TOKEN=$(cat ${TOKEN_FILE})
174 + # ── Process a group and all its subgroups recursively ────────────────
175 + process_group() {
176 + local group_id="$1"
79 177
80 - for PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups?per_page=100 2>/dev/null| jq -r '."x-total-pages"[0]'))
81 - do
82 - for GROUP in $(curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups?per_page=100&page=${PAGE}" | jq -r '.[].id')
83 - do
84 - for P_PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups/${GROUP}/projects?per_page=100 2>/dev/null| jq -r '."x-total-pages"[0]'))
85 - do
86 - for PROJECT in $(curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups/${GROUP}/projects?per_page=100&page=${P_PAGE}" | jq -r '.[].id')
87 - do
88 - clone_project ${PROJECT}
89 - done
90 - done
91 - for SG_PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups/${GROUP}/subgroups?per_page=100 2>/dev/null | jq -r '."x-total-pages"[0]'))
92 - do
93 - for SUBGROUP in $(curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups/${GROUP}/subgroups?per_page=100&page=${SG_PAGE}" | jq -r '.[].id')
94 - do
95 - for P_SG_PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups/${SUBGROUP}/projects?per_page=100 2>/dev/null| jq -r '."x-total-pages"[0]'))
96 - do
97 - for PROJECT in $(curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups/${SUBGROUP}/projects?per_page=100&page=${P_SG_PAGE}" | jq -r '.[].id')
98 - do
99 - clone_project ${PROJECT}
100 - done
101 - done
102 - done
178 + clone_group_projects "$group_id"
179 +
180 + # Collect subgroup IDs, then recurse
181 + local subgroup_ids=()
182 + while IFS= read -r sid; do
183 + [[ -z "$sid" ]] && continue
184 + subgroup_ids+=("$sid")
185 + done < <(
186 + local total
187 + total="$(gitlab_total_pages "${GITLAB_URL}/api/v4/groups/${group_id}/subgroups?per_page=100")"
188 + for (( p = 1; p <= total; p++ )); do
189 + gitlab_get "${GITLAB_URL}/api/v4/groups/${group_id}/subgroups?per_page=100&page=${p}" \
190 + | jq -r '.[].id'
103 191 done
192 + )
193 +
194 + local sid
195 + for sid in "${subgroup_ids[@]}"; do
196 + log_info "Entering subgroup ${sid}"
197 + process_group "$sid"
104 198 done
105 - done
106 - echo "OK"
199 + }
200 +
201 + # ── Main ─────────────────────────────────────────────────────────────
202 + main() {
203 + log_info "GitLab URL: ${GITLAB_URL}"
204 + log_info "Token file: ${TOKEN_FILE}"
205 + echo
206 +
207 + # Iterate over every top-level group visible to the token
208 + gitlab_paginate \
209 + "${GITLAB_URL}/api/v4/groups?per_page=100&top_level_only=true" \
210 + '.[].id' process_group
211 +
212 + echo
213 + log_ok "Done — cloned ${CLONE_COUNT} repo(s), skipped ${SKIP_COUNT} already present"
214 + }
215 +
216 + main

botanikanet revised this gist 8 months ago. Go to revision

No changes

botanikanet revised this gist 8 months ago. Go to revision

1 file changed, 106 insertions

get_all_src.sh(file created)

@@ -0,0 +1,106 @@
1 + #!/bin/bash
2 + set -ex
3 +
4 + usage() {
5 + echo "basename ${0} get all src from Gitlab"
6 + echo "--help | -h Print this help"
7 + echo "--token-file= | -t <file> Set file with Gitlab token"
8 + echo "--gitlab-url=* | -g <url> Set Gtilab URL"
9 + exit 1
10 + }
11 +
12 + # Map long options to short ones and rebuild argv
13 + translated=()
14 + while (( $# )); do
15 + case "$1" in
16 + --help) translated+=("-h"); shift ;;
17 + --token-file=*) translated+=("-t" "${1#*=}"); shift ;;
18 + --gitlab-url=*) translated+=("-g" "${1#*=}"); shift ;;
19 + --) translated+=("--"); shift; translated+=("$@"); set -- ;;
20 + *) translated+=("$1"); shift ;;
21 + esac
22 + done
23 + set -- "${translated[@]}"
24 +
25 + declare GITLAB_URL="https://git.mws-team.ru"
26 + declare TOKEN_FILE=""
27 +
28 + while getopts ":ht:g:" opt; do
29 + case "$opt" in
30 + h) usage ;;
31 + t) TOKEN_FILE=$OPTARG ;;
32 + g) GITLAB_URL=$OPTARG ;;
33 + :) echo "Missing arg for -$OPTARG"; usage ;;
34 + \?) echo "Invalid option: -$OPTARG"; usage ;;
35 + esac
36 + done
37 +
38 + function checks {
39 + if [ -z "${TOKEN_FILE}" ]
40 + then
41 + echo "Missing token file"
42 + usage
43 + fi
44 +
45 + if [ -z "${GITLAB_URL}" ]
46 + then
47 + echo "Missing gitlab url"
48 + usage
49 + fi
50 +
51 + if [ ! -f "${TOKEN_FILE}" ]
52 + then
53 + echo "Token file not found"
54 + usage
55 + fi
56 + }
57 +
58 + function get_subgroups {
59 + for PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups/${1}/subgroups?per_page=100 2>/dev/null| jq -r '."x-total-pages"[0]'))
60 + do
61 + curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups/${1}/subgroups?per_page=100&page=${PAGE}" | jq -r '.[].id'
62 + done
63 + }
64 +
65 + function clone_project {
66 + PROJECT_INFO=$(curl --silent --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/projects/${1} | jq .)
67 + echo "INFO: Check project $(echo ${PROJECT_INFO} | jq -r '.path_with_namespace')"
68 + if [ ! -d "$(echo ${PROJECT_INFO} | jq -r '.path_with_namespace')" ]
69 + then
70 + echo "INFO: Clone project $(echo ${PROJECT_INFO} | jq -r '.path_with_namespace')"
71 + mkdir -p $(echo ${PROJECT_INFO} | jq -r '.path_with_namespace')
72 + git clone $(echo ${PROJECT_INFO} | jq -r '. | "\(.ssh_url_to_repo) \(.path_with_namespace)"')
73 + fi
74 + }
75 +
76 + checks
77 +
78 + TOKEN=$(cat ${TOKEN_FILE})
79 +
80 + for PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups?per_page=100 2>/dev/null| jq -r '."x-total-pages"[0]'))
81 + do
82 + for GROUP in $(curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups?per_page=100&page=${PAGE}" | jq -r '.[].id')
83 + do
84 + for P_PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups/${GROUP}/projects?per_page=100 2>/dev/null| jq -r '."x-total-pages"[0]'))
85 + do
86 + for PROJECT in $(curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups/${GROUP}/projects?per_page=100&page=${P_PAGE}" | jq -r '.[].id')
87 + do
88 + clone_project ${PROJECT}
89 + done
90 + done
91 + for SG_PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups/${GROUP}/subgroups?per_page=100 2>/dev/null | jq -r '."x-total-pages"[0]'))
92 + do
93 + for SUBGROUP in $(curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups/${GROUP}/subgroups?per_page=100&page=${SG_PAGE}" | jq -r '.[].id')
94 + do
95 + for P_SG_PAGE in $(seq 1 $(curl -X HEAD -w '%{header_json}' --header "Authorization: Bearer ${TOKEN}" ${GITLAB_URL}/api/v4/groups/${SUBGROUP}/projects?per_page=100 2>/dev/null| jq -r '."x-total-pages"[0]'))
96 + do
97 + for PROJECT in $(curl --silent --header "Authorization: Bearer ${TOKEN}" "${GITLAB_URL}/api/v4/groups/${SUBGROUP}/projects?per_page=100&page=${P_SG_PAGE}" | jq -r '.[].id')
98 + do
99 + clone_project ${PROJECT}
100 + done
101 + done
102 + done
103 + done
104 + done
105 + done
106 + echo "OK"
Newer Older