botanikanet / get_group_src.sh

#!/bin/bash set -euo pipefail # ── Colors ─────────────────────────────────────────────────────────── RED='\033[0;31m' GREEN='\033[0;32m' YELLOW='\033[1;33m' CYAN='\033[0;36m' NC='\033[0m' # ── Logging ────────────────────────────────────────────────────────── log_info() { printf "${CYAN}[INFO]${NC} %s\n" "$*"; } log_warn() { printf "${YELLOW}[WARN]${NC} %s\n" "$*"; } log_ok() { printf "${GREEN}[ OK ]${NC} %s\n" "$*"; } log_error() { printf "${RED}[ERR]${NC} %s\n" "$*" >&2; } # ── Usage ──────────────────────────────────────────────────────────── usage() { cat <<EOF Usage: $(basename "$0") [OPTIONS] Clone all projects from a GitLab group (including subgroups) preserving the path_with_namespace directory structure. Options: -h, --help Show this help -t, --token-file=FILE File containing the GitLab personal access token -g, --gitlab-url=URL GitLab instance URL (default: https://git.mws-team.ru) -i, --group-id=ID GitLab group ID to clone Examples: $(basename "$0") -t ~/.gitlab_token -i 42 $(basename "$0") --token-file=token.txt --gitlab-url=https://gitlab.com --group-id=123 EOF exit 1 } # ── Dependency check ───────────────────────────────────────────────── for cmd in curl jq git; do if ! command -v "$cmd" &>/dev/null; then log_error "Required command not found: ${cmd}" exit 1 fi done # ── Parse arguments ────────────────────────────────────────────────── GITLAB_URL="https://git.mws-team.ru" TOKEN_FILE="" GROUP="" translated=() while (( $# )); do case "$1" in --help) translated+=("-h"); shift ;; --token-file=*) translated+=("-t" "${1#*=}"); shift ;; --gitlab-url=*) translated+=("-g" "${1#*=}"); shift ;; --group-id=*) translated+=("-i" "${1#*=}"); shift ;; --) translated+=("--"); shift; translated+=("$@"); break ;; *) translated+=("$1"); shift ;; esac done set -- "${translated[@]}" while getopts ":ht:g:i:" opt; do case "$opt" in h) usage ;; t) TOKEN_FILE="$OPTARG" ;; g) GITLAB_URL="$OPTARG" ;; i) GROUP="$OPTARG" ;; :) log_error "Missing argument for -${OPTARG}"; usage ;; \?) log_error "Invalid option: -${OPTARG}"; usage ;; esac done shift $((OPTIND - 1)) # ── Validate inputs ───────────────────────────────────────────────── [[ -z "$TOKEN_FILE" ]] && { log_error "Missing --token-file"; usage; } [[ -z "$GITLAB_URL" ]] && { log_error "Missing --gitlab-url"; usage; } [[ -z "$GROUP" ]] && { log_error "Missing --group-id"; usage; } [[ ! -f "$TOKEN_FILE" ]] && { log_error "Token file not found: ${TOKEN_FILE}"; exit 1; } # Strip trailing slash from URL GITLAB_URL="${GITLAB_URL%/}" TOKEN="$(<"$TOKEN_FILE")" readonly TOKEN GITLAB_URL GROUP # ── GitLab API helpers ─────────────────────────────────────────────── # Perform an authenticated GET request and return the JSON body. gitlab_get() { local url="$1" curl --silent --fail --header "Authorization: Bearer ${TOKEN}" "$url" } # Return total number of pages for a paginated endpoint. # Usage: gitlab_total_pages "https://gitlab.com/api/v4/..." gitlab_total_pages() { local url="$1" local headers headers="$(curl -X HEAD --silent -w '%{header_json}' \ --header "Authorization: Bearer ${TOKEN}" \ -o /dev/null "$url" 2>/dev/null)" || true local pages pages="$(echo "$headers" | jq -r '."x-total-pages"[0] // "1"')" # Fallback to 1 if empty or null [[ -z "$pages" || "$pages" == "null" ]] && pages=1 echo "$pages" } # Iterate over all pages of a paginated endpoint, streaming JSON arrays. # Calls a callback function with each item's specified jq field. # Usage: gitlab_paginate "/api/v4/groups/42/projects?per_page=100" ".[].id" callback_fn gitlab_paginate() { local endpoint="$1" local jq_filter="$2" local callback="$3" local extra_params="${4:-}" local base_url="${GITLAB_URL}${endpoint}" [[ -n "$extra_params" ]] && base_url="${base_url}&${extra_params}" local total_pages total_pages="$(gitlab_total_pages "$base_url")" local page for (( page = 1; page <= total_pages; page++ )); do local response response="$(gitlab_get "${base_url}&page=${page}")" || { log_warn "API request failed: ${base_url}&page=${page}" continue } local item while IFS= read -r item; do [[ -z "$item" ]] && continue "$callback" "$item" done < <(echo "$response" | jq -r "$jq_filter") done } # ── Clone a single project ────────────────────────────────────────── clone_project() { local project_id="$1" local project_json project_json="$(gitlab_get "${GITLAB_URL}/api/v4/projects/${project_id}")" || { log_warn "Failed to fetch project ${project_id}" return } local path_ns ssh_url path_ns="$(echo "$project_json" | jq -r '.path_with_namespace')" ssh_url="$(echo "$project_json" | jq -r '.ssh_url_to_repo')" if [[ -z "$path_ns" || "$path_ns" == "null" ]]; then log_warn "Could not determine path for project ${project_id} — skipping" return fi if [[ -d "$path_ns" ]]; then log_info "Already cloned: ${path_ns}" return fi log_info "Cloning: ${path_ns}" local parent_dir parent_dir="$(dirname "$path_ns")" mkdir -p "$parent_dir" git clone "$ssh_url" "$path_ns" --quiet || log_warn "git clone failed for ${path_ns}" } # ── Clone all projects in a group ──────────────────────────────────── clone_group_projects() { local group_id="$1" log_info "Cloning projects in group ${group_id}" gitlab_paginate "/api/v4/groups/${group_id}/projects?per_page=100" \ '.[].id' clone_project "archived=no" } # ── Process a group and all its subgroups recursively ──────────────── process_group() { local group_id="$1" # Clone direct projects clone_group_projects "$group_id" # Recurse into subgroups local subgroup_ids=() while IFS= read -r sid; do [[ -z "$sid" ]] && continue subgroup_ids+=("$sid") done < <( local total total="$(gitlab_total_pages "${GITLAB_URL}/api/v4/groups/${group_id}/subgroups?per_page=100")" for (( p = 1; p <= total; p++ )); do gitlab_get "${GITLAB_URL}/api/v4/groups/${group_id}/subgroups?per_page=100&page=${p}" \ | jq -r '.[].id' done ) for sid in "${subgroup_ids[@]}"; do log_info "Entering subgroup ${sid}" process_group "$sid" done } # ── Main ───────────────────────────────────────────────────────────── main() { log_info "GitLab URL: ${GITLAB_URL}" log_info "Group ID: ${GROUP}" log_info "Token file: ${TOKEN_FILE}" echo process_group "$GROUP" echo log_ok "Done" } main