botanikanet / get_all_src.sh

#!/bin/bash set -euo pipefail # ── Colors ─────────────────────────────────────────────────────────── RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' CYAN='\033[0;36m' NC='\033[0m' # ── Logging ────────────────────────────────────────────────────────── log_info() { printf "${CYAN}[INFO]${NC} %s\n" "$*"; } log_warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$*"; } log_ok() { printf "${GREEN}[ OK ]${NC} %s\n" "$*"; } log_error() { printf "${RED}[ERR]${NC} %s\n" "$*" >&2; } # ── Usage ──────────────────────────────────────────────────────────── usage() { cat <<EOF Usage: $(basename "$0") [OPTIONS] Clone every project from every group (and subgroups, recursively) on a GitLab instance, preserving the path_with_namespace directory structure. Options: -h, --help Show this help -t, --token-file=FILE File containing the GitLab personal access token -g, --gitlab-url=URL GitLab instance URL (default: https://git.mws-team.ru) Examples: $(basename "$0") -t ~/.gitlab_token $(basename "$0") --token-file=token.txt --gitlab-url=https://gitlab.com EOF exit 1 } # ── Dependency check ───────────────────────────────────────────────── for cmd in curl jq git; do if ! command -v "$cmd" &>/dev/null; then log_error "Required command not found: ${cmd}" exit 1 fi done # ── Parse arguments ────────────────────────────────────────────────── GITLAB_URL="https://git.mws-team.ru" TOKEN_FILE="" translated=() while (( $# )); do case "$1" in --help) translated+=("-h"); shift ;; --token-file=*) translated+=("-t" "${1#*=}"); shift ;; --gitlab-url=*) translated+=("-g" "${1#*=}"); shift ;; --) translated+=("--"); shift; translated+=("$@"); break ;; *) translated+=("$1"); shift ;; esac done set -- "${translated[@]}" while getopts ":ht:g:" opt; do case "$opt" in h) usage ;; t) TOKEN_FILE="$OPTARG" ;; g) GITLAB_URL="$OPTARG" ;; :) log_error "Missing argument for -${OPTARG}"; usage ;; \?) log_error "Invalid option: -${OPTARG}"; usage ;; esac done shift $((OPTIND - 1)) # ── Validate inputs ───────────────────────────────────────────────── [[ -z "$TOKEN_FILE" ]] && { log_error "Missing --token-file"; usage; } [[ -z "$GITLAB_URL" ]] && { log_error "Missing --gitlab-url"; usage; } [[ ! -f "$TOKEN_FILE" ]] && { log_error "Token file not found: ${TOKEN_FILE}"; exit 1; } GITLAB_URL="${GITLAB_URL%/}" TOKEN="$(<"$TOKEN_FILE")" readonly TOKEN GITLAB_URL # ── GitLab API helpers ─────────────────────────────────────────────── # Authenticated GET request; returns JSON body. gitlab_get() { curl --silent --fail --header "Authorization: Bearer ${TOKEN}" "$1" } # Return total page count for a paginated endpoint. gitlab_total_pages() { local headers headers="$(curl -X HEAD --silent -w '%{header_json}' \ --header "Authorization: Bearer ${TOKEN}" \ -o /dev/null "$1" 2>/dev/null)" || true local pages pages="$(echo "$headers" | jq -r '."x-total-pages"[0] // "1"')" [[ -z "$pages" || "$pages" == "null" ]] && pages=1 echo "$pages" } # Iterate all pages of an endpoint, call a callback with each extracted value. # Usage: gitlab_paginate "URL?per_page=100" '.[].id' callback_fn gitlab_paginate() { local base_url="$1" local jq_filter="$2" local callback="$3" local total_pages total_pages="$(gitlab_total_pages "$base_url")" local page for (( page = 1; page <= total_pages; page++ )); do local response response="$(gitlab_get "${base_url}&page=${page}")" || { log_warn "API request failed: ${base_url}&page=${page}" continue } local item while IFS= read -r item; do [[ -z "$item" ]] && continue "$callback" "$item" done < <(echo "$response" | jq -r "$jq_filter") done } # ── Clone a single project ────────────────────────────────────────── CLONE_COUNT=0 SKIP_COUNT=0 clone_project() { local project_id="$1" local project_json project_json="$(gitlab_get "${GITLAB_URL}/api/v4/projects/${project_id}")" || { log_warn "Failed to fetch project ${project_id}" return } local path_ns ssh_url path_ns="$(echo "$project_json" | jq -r '.path_with_namespace')" ssh_url="$(echo "$project_json" | jq -r '.ssh_url_to_repo')" if [[ -z "$path_ns" || "$path_ns" == "null" ]]; then log_warn "Could not determine path for project ${project_id} — skipping" return fi if [[ -d "$path_ns" ]]; then log_info "Already cloned: ${path_ns}" (( SKIP_COUNT++ )) || true return fi log_info "Cloning: ${path_ns}" mkdir -p "$(dirname "$path_ns")" if git clone --quiet "$ssh_url" "$path_ns"; then log_ok "Cloned: ${path_ns}" (( CLONE_COUNT++ )) || true else log_warn "git clone failed for ${path_ns}" fi } # ── Clone all projects in a group ──────────────────────────────────── clone_group_projects() { local group_id="$1" gitlab_paginate \ "${GITLAB_URL}/api/v4/groups/${group_id}/projects?per_page=100&archived=no" \ '.[].id' clone_project } # ── Process a group and all its subgroups recursively ──────────────── process_group() { local group_id="$1" clone_group_projects "$group_id" # Collect subgroup IDs, then recurse local subgroup_ids=() while IFS= read -r sid; do [[ -z "$sid" ]] && continue subgroup_ids+=("$sid") done < <( local total total="$(gitlab_total_pages "${GITLAB_URL}/api/v4/groups/${group_id}/subgroups?per_page=100")" for (( p = 1; p <= total; p++ )); do gitlab_get "${GITLAB_URL}/api/v4/groups/${group_id}/subgroups?per_page=100&page=${p}" \ | jq -r '.[].id' done ) local sid for sid in "${subgroup_ids[@]}"; do log_info "Entering subgroup ${sid}" process_group "$sid" done } # ── Main ───────────────────────────────────────────────────────────── main() { log_info "GitLab URL: ${GITLAB_URL}" log_info "Token file: ${TOKEN_FILE}" echo # Iterate over every top-level group visible to the token gitlab_paginate \ "${GITLAB_URL}/api/v4/groups?per_page=100&top_level_only=true" \ '.[].id' process_group echo log_ok "Done — cloned ${CLONE_COUNT} repo(s), skipped ${SKIP_COUNT} already present" } main