459 lines
13 KiB
Bash
Executable File
459 lines
13 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
# Wait until the forge pipeline for an exact commit SHA finishes.
|
|
# Does not trigger a pipeline. Does not print tokens.
|
|
#
|
|
# Usage:
|
|
# wait_ci.sh --sha SHA [--root DIR] [--timeout SEC]
|
|
#
|
|
# Supported forges:
|
|
# GitHub Actions (gh), GitLab CI (glab), and Gitea Actions (curl + REST API).
|
|
# Set BUILDER_CI_FORGE=gitea for a custom Gitea host that does not contain
|
|
# "gitea" in its hostname. GITEA_API_URL may override the /api/v1 endpoint.
|
|
#
|
|
# Exit:
|
|
# 0 pipeline succeeded; prints "OK pipeline=<url> sha=<sha>"
|
|
# 1 no pipeline for this SHA, or a job failed/canceled
|
|
# 2 usage / bad arguments
|
|
# 3 forge CLI missing, or it cannot read this project
|
|
|
|
usage() {
|
|
sed -n '2,18p' "$0"
|
|
}
|
|
|
|
sha=
|
|
root=
|
|
timeout_sec=1200
|
|
poll_sec=${BUILDER_CI_POLL_SEC:-10}
|
|
appear_sec=${BUILDER_CI_APPEAR_SEC:-90}
|
|
|
|
while (($#)); do
|
|
case "$1" in
|
|
--sha) [[ $# -ge 2 ]] || { echo "Error: --sha needs a value" >&2; exit 2; }; sha=$2; shift 2 ;;
|
|
--root) [[ $# -ge 2 ]] || { echo "Error: --root needs a value" >&2; exit 2; }; root=$2; shift 2 ;;
|
|
--timeout) [[ $# -ge 2 ]] || { echo "Error: --timeout needs a value" >&2; exit 2; }; timeout_sec=$2; shift 2 ;;
|
|
-h|--help) usage; exit 0 ;;
|
|
*) echo "Error: unknown argument: $1" >&2; usage >&2; exit 2 ;;
|
|
esac
|
|
done
|
|
|
|
if [[ -z "$root" ]]; then
|
|
root=$(git rev-parse --show-toplevel 2>/dev/null || true)
|
|
fi
|
|
if [[ -z "$root" || ! -d "$root/.git" && ! -f "$root/.git" ]]; then
|
|
echo "Error: --root must be a git project" >&2
|
|
exit 2
|
|
fi
|
|
if [[ -z "$sha" ]]; then
|
|
echo "Error: --sha is required" >&2
|
|
exit 2
|
|
fi
|
|
sha=$(git -C "$root" rev-parse "$sha")
|
|
|
|
# Load only builder configuration. Shell values win and values are never echoed.
|
|
if [[ -f "$root/.env.builder" ]]; then
|
|
while IFS='=' read -r key value; do
|
|
key=${key%%[[:space:]]*}
|
|
[[ -z "$key" || "$key" == \#* ]] && continue
|
|
[[ "$key" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]] || continue
|
|
if [[ -n "${!key:-}" ]]; then
|
|
continue
|
|
fi
|
|
value=${value%\"}; value=${value#\"}; value=${value%\'}; value=${value#\'}
|
|
[[ -z "$value" ]] && continue
|
|
printf -v "$key" '%s' "$value"
|
|
export "$key"
|
|
done < <(grep -v '^[[:space:]]*$' "$root/.env.builder")
|
|
fi
|
|
|
|
python_json() {
|
|
python3 -c "$1" "${@:2}"
|
|
}
|
|
|
|
parse_origin() {
|
|
python_json '
|
|
import os, re
|
|
url = os.environ["WAIT_CI_ORIGIN"].strip()
|
|
url = re.sub(r"^git@([^:]+):", r"ssh://git@\1/", url)
|
|
url = re.sub(r"^ssh://git@([^/]+):", r"ssh://git@\1/", url)
|
|
if re.match(r"^https?://", url):
|
|
m = re.match(r"^(https?)://([^/]+)(?:/(.*))?$", url)
|
|
scheme, host, path = (m.group(1), m.group(2), m.group(3) or "") if m else ("", "", "")
|
|
else:
|
|
m = re.match(r"^ssh://git@([^/]+)/(.+)$", url)
|
|
scheme, host, path = ("https", m.group(1), m.group(2)) if m else ("", "", "")
|
|
path = re.sub(r"\.git$", "", path)
|
|
override = os.environ.get("BUILDER_CI_FORGE", "").strip().lower()
|
|
if override in ("github", "gitlab", "gitea"):
|
|
kind = override
|
|
elif host == "github.com" or host.endswith(".github.com"):
|
|
kind = "github"
|
|
elif ("gitea" in host.lower() or os.environ.get("GITEA_TOKEN") or
|
|
os.environ.get("GITEA_API_URL")):
|
|
kind = "gitea"
|
|
else:
|
|
kind = "gitlab"
|
|
print(kind, host, path, (scheme or "https") + "://" + host)
|
|
'
|
|
}
|
|
|
|
origin=$(git -C "$root" remote get-url origin 2>/dev/null || true)
|
|
if [[ -z "$origin" ]]; then
|
|
echo "Error: no origin remote" >&2
|
|
exit 2
|
|
fi
|
|
|
|
export WAIT_CI_ORIGIN=$origin
|
|
read -r forge_kind forge_host forge_path forge_base <<<"$(parse_origin)"
|
|
if [[ -z "$forge_kind" || -z "$forge_path" ]]; then
|
|
echo "Error: cannot parse origin: $origin" >&2
|
|
exit 2
|
|
fi
|
|
export WAIT_CI_PATH=$forge_path
|
|
export WAIT_CI_SHA=$sha
|
|
|
|
gitlab_pipelines() {
|
|
local encoded
|
|
encoded=$(python_json 'import os,urllib.parse; print(urllib.parse.quote(os.environ["WAIT_CI_PATH"], safe=""))')
|
|
glab api "projects/${encoded}/pipelines?sha=${sha}&per_page=20"
|
|
}
|
|
|
|
gitlab_jobs() {
|
|
local encoded pid=$1
|
|
encoded=$(python_json 'import os,urllib.parse; print(urllib.parse.quote(os.environ["WAIT_CI_PATH"], safe=""))')
|
|
glab api "projects/${encoded}/pipelines/${pid}/jobs"
|
|
}
|
|
|
|
github_runs() {
|
|
gh run list --repo "$forge_path" --commit "$sha" --limit 20 \
|
|
--json databaseId,status,conclusion,url,headSha
|
|
}
|
|
|
|
gitea_api_base=${GITEA_API_URL:-$forge_base/api/v1}
|
|
gitea_api_base=${gitea_api_base%/}
|
|
if [[ "$gitea_api_base" != */api/v1 ]]; then
|
|
gitea_api_base+="/api/v1"
|
|
fi
|
|
gitea_repo_path=$(python_json 'import os,urllib.parse; print(urllib.parse.quote(os.environ["WAIT_CI_PATH"], safe="/"))')
|
|
|
|
gitea_curl() {
|
|
local url=$1
|
|
if [[ -n "${GITEA_TOKEN:-}" ]]; then
|
|
# Process substitution keeps the token out of curl's argument list and logs.
|
|
curl -fsS --retry 2 --connect-timeout 10 --max-time 30 \
|
|
--config <(printf 'header = "Authorization: token %s"\n' "$GITEA_TOKEN") "$url"
|
|
else
|
|
curl -fsS --retry 2 --connect-timeout 10 --max-time 30 "$url"
|
|
fi
|
|
}
|
|
|
|
gitea_runs() {
|
|
gitea_curl "${gitea_api_base}/repos/${gitea_repo_path}/actions/runs?head_sha=${sha}&limit=20&page=1"
|
|
}
|
|
|
|
gitea_jobs() {
|
|
gitea_curl "${gitea_api_base}/repos/${gitea_repo_path}/actions/runs/$1/jobs?limit=100&page=1"
|
|
}
|
|
|
|
summarize_gitlab() {
|
|
python_json '
|
|
import json, sys
|
|
try:
|
|
data = json.loads(sys.stdin.read())
|
|
except json.JSONDecodeError:
|
|
print("error")
|
|
raise SystemExit(0)
|
|
if isinstance(data, dict) and data.get("message"):
|
|
print("auth", data.get("message", "")[:80].replace(" ", "_"))
|
|
raise SystemExit(0)
|
|
if not isinstance(data, list):
|
|
print("error")
|
|
raise SystemExit(0)
|
|
if not data:
|
|
print("empty")
|
|
raise SystemExit(0)
|
|
p = data[0]
|
|
print(p.get("id"), p.get("status"), p.get("web_url", ""))
|
|
'
|
|
}
|
|
|
|
summarize_jobs() {
|
|
python_json '
|
|
import json, sys
|
|
data = json.loads(sys.stdin.read())
|
|
parts = []
|
|
for j in data if isinstance(data, list) else []:
|
|
parts.append("%s=%s" % (j.get("name"), j.get("status")))
|
|
print(",".join(parts))
|
|
'
|
|
}
|
|
|
|
summarize_github() {
|
|
python_json '
|
|
import json, sys
|
|
data = json.loads(sys.stdin.read())
|
|
if not data:
|
|
print("empty")
|
|
raise SystemExit(0)
|
|
r = data[0]
|
|
status = r.get("status")
|
|
conc = r.get("conclusion")
|
|
if status == "completed" and conc == "success":
|
|
st = "success"
|
|
elif status == "completed":
|
|
st = conc or "failed"
|
|
else:
|
|
st = status or "running"
|
|
print(r.get("databaseId"), st, r.get("url", ""))
|
|
'
|
|
}
|
|
|
|
summarize_gitea_runs() {
|
|
python_json '
|
|
import json, sys
|
|
try:
|
|
data = json.loads(sys.stdin.read())
|
|
except json.JSONDecodeError:
|
|
print("error")
|
|
raise SystemExit(0)
|
|
if isinstance(data, dict) and data.get("message") and not (data.get("workflow_runs") or data.get("runs")):
|
|
print("auth", data.get("message", "")[:80].replace(" ", "_"))
|
|
raise SystemExit(0)
|
|
if isinstance(data, dict):
|
|
data = data.get("workflow_runs", data.get("runs", []))
|
|
if not isinstance(data, list):
|
|
print("error")
|
|
raise SystemExit(0)
|
|
want = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
for run in data:
|
|
head = run.get("head_sha") or run.get("headSha") or (run.get("head_commit") or {}).get("id")
|
|
if not head or head.lower() != want.lower():
|
|
continue
|
|
run_id = run.get("id") or run.get("database_id")
|
|
if run_id is None:
|
|
continue
|
|
status = str(run.get("status") or "").lower()
|
|
conclusion = str(run.get("conclusion") or run.get("result") or "").lower()
|
|
url = run.get("html_url") or run.get("web_url") or run.get("url") or "-"
|
|
print("%s\t%s\t%s\t%s" % (run_id, status, conclusion, url))
|
|
' "$sha"
|
|
}
|
|
|
|
summarize_gitea_jobs() {
|
|
python_json '
|
|
import json, sys
|
|
try:
|
|
data = json.loads(sys.stdin.read())
|
|
except json.JSONDecodeError:
|
|
print("error")
|
|
raise SystemExit(0)
|
|
if isinstance(data, dict) and data.get("message") and not data.get("jobs"):
|
|
print("auth", data.get("message", "")[:80].replace(" ", "_"))
|
|
raise SystemExit(0)
|
|
if isinstance(data, dict):
|
|
data = data.get("jobs", [])
|
|
if not isinstance(data, list):
|
|
print("error")
|
|
raise SystemExit(0)
|
|
want = sys.argv[1] if len(sys.argv) > 1 else ""
|
|
parts = []
|
|
state = "success" if data else "empty"
|
|
for job in data:
|
|
head = job.get("head_sha") or job.get("headSha")
|
|
if head and want and head.lower() != want.lower():
|
|
continue
|
|
name = str(job.get("name") or job.get("id") or "job").replace(" ", "_")
|
|
status = str(job.get("status") or "").lower()
|
|
result = str(job.get("conclusion") or job.get("result") or "").lower()
|
|
value = result or status or "unknown"
|
|
parts.append("%s=%s" % (name, value))
|
|
if value in ("failure", "failed", "cancelled", "canceled", "skipped", "timed_out", "action_required"):
|
|
state = "failed"
|
|
elif value not in ("success", "succeeded", "completed"):
|
|
state = "running"
|
|
if not parts:
|
|
state = "empty"
|
|
print("%s\t%s" % (state, ",".join(parts)))
|
|
' "$sha"
|
|
}
|
|
|
|
gitea_status_line() {
|
|
local raw runs
|
|
if ! raw=$(gitea_runs); then
|
|
echo "error"
|
|
return 0
|
|
fi
|
|
runs=$(printf '%s' "$raw" | summarize_gitea_runs)
|
|
if [[ "$runs" == auth* || "$runs" == error ]]; then
|
|
printf '%s\n' "$runs"
|
|
return 0
|
|
fi
|
|
if [[ -z "$runs" ]]; then
|
|
echo "empty"
|
|
return 0
|
|
fi
|
|
|
|
local overall=success ids= urls= jobs_all= first=1
|
|
while IFS=$'\t' read -r id run_status conclusion url; do
|
|
[[ -z "$id" ]] && continue
|
|
local jobs job_line job_state job_summary
|
|
if ! jobs=$(gitea_jobs "$id"); then
|
|
echo "error"
|
|
return 0
|
|
fi
|
|
job_line=$(printf '%s' "$jobs" | summarize_gitea_jobs "$sha")
|
|
job_state=${job_line%%$'\t'*}
|
|
job_summary=${job_line#*$'\t'}
|
|
if [[ "$job_state" == auth* || "$job_state" == error ]]; then
|
|
printf '%s\n' "$job_state"
|
|
return 0
|
|
fi
|
|
if [[ "$first" -eq 0 ]]; then
|
|
ids+=","; urls+=","; jobs_all+=","
|
|
fi
|
|
ids+="$id"; urls+="$url"; jobs_all+="$job_summary"; first=0
|
|
if [[ "$run_status" =~ ^(failure|failed|cancelled|canceled|skipped|timed_out|action_required)$ || "$conclusion" =~ ^(failure|failed|cancelled|canceled|skipped|timed_out|action_required)$ || "$job_state" == failed ]]; then
|
|
overall=failed
|
|
elif [[ "$overall" == success && ( "$job_state" == running || "$job_state" == empty || ( "$run_status" != completed && "$run_status" != success ) ) ]]; then
|
|
overall=running
|
|
fi
|
|
done <<<"$runs"
|
|
printf '%s %s %s %s\n' "$ids" "$overall" "$urls" "$jobs_all"
|
|
}
|
|
|
|
if [[ "$forge_kind" == github ]]; then
|
|
if ! command -v gh >/dev/null 2>&1; then
|
|
echo "Error: gh is required to watch GitHub Actions for $forge_host" >&2
|
|
exit 3
|
|
fi
|
|
elif [[ "$forge_kind" == gitlab ]]; then
|
|
if ! command -v glab >/dev/null 2>&1; then
|
|
echo "Error: glab is required to watch GitLab CI for $forge_host" >&2
|
|
exit 3
|
|
fi
|
|
else
|
|
if ! command -v curl >/dev/null 2>&1; then
|
|
echo "Error: curl is required to watch Gitea Actions for $forge_host" >&2
|
|
exit 3
|
|
fi
|
|
fi
|
|
|
|
started=$(date +%s)
|
|
first_seen=0
|
|
empty_since=$started
|
|
|
|
probe_auth() {
|
|
local out rc kind
|
|
if [[ "$forge_kind" == github ]]; then
|
|
set +e
|
|
out=$(github_runs 2>&1)
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -ne 0 ]]; then
|
|
echo "Error: gh cannot list runs for $forge_path" >&2
|
|
echo "$out" | sed '/token/Id; /Bearer/d' >&2
|
|
exit 3
|
|
fi
|
|
return
|
|
fi
|
|
if [[ "$forge_kind" == gitlab ]]; then
|
|
set +e
|
|
out=$(gitlab_pipelines 2>&1)
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -ne 0 ]]; then
|
|
echo "Error: glab cannot list pipelines for $forge_path on $forge_host" >&2
|
|
echo "$out" | sed '/token/Id; /Bearer/d' >&2
|
|
exit 3
|
|
fi
|
|
kind=$(printf '%s' "$out" | summarize_gitlab | awk '{print $1}')
|
|
if [[ "$kind" == auth ]]; then
|
|
echo "Error: glab is authenticated but cannot read $forge_path" >&2
|
|
exit 3
|
|
fi
|
|
return
|
|
fi
|
|
set +e
|
|
out=$(gitea_runs 2>&1)
|
|
rc=$?
|
|
set -e
|
|
if [[ $rc -ne 0 ]]; then
|
|
echo "Error: curl cannot read Gitea Actions for $forge_path on $forge_host" >&2
|
|
echo "$out" | sed '/token/Id; /Bearer/d; /Authorization/Id' >&2
|
|
exit 3
|
|
fi
|
|
kind=$(printf '%s' "$out" | summarize_gitea_runs | awk '{print $1}')
|
|
if [[ "$kind" == auth ]]; then
|
|
echo "Error: Gitea API rejected access to $forge_path" >&2
|
|
exit 3
|
|
fi
|
|
}
|
|
|
|
probe_auth
|
|
|
|
status_line() {
|
|
if [[ "$forge_kind" == github ]]; then
|
|
github_runs | summarize_github
|
|
elif [[ "$forge_kind" == gitlab ]]; then
|
|
gitlab_pipelines | summarize_gitlab
|
|
else
|
|
gitea_status_line
|
|
fi
|
|
}
|
|
|
|
while true; do
|
|
now=$(date +%s)
|
|
if (( now - started > timeout_sec )); then
|
|
echo "FAIL reason=timeout sha=$sha" >&2
|
|
exit 1
|
|
fi
|
|
line=$(status_line)
|
|
set -- $line
|
|
kind=${1:-empty}
|
|
|
|
if [[ "$kind" == empty ]]; then
|
|
if (( now - empty_since >= appear_sec )); then
|
|
echo "FAIL reason=no_pipeline sha=$sha origin=$forge_host/$forge_path" >&2
|
|
echo "This SHA has no CI pipeline. Tag or push a ref the project actually builds." >&2
|
|
exit 1
|
|
fi
|
|
sleep "$poll_sec"
|
|
continue
|
|
fi
|
|
if [[ "$kind" == error || "$kind" == auth ]]; then
|
|
echo "Error: unexpected forge response for $forge_path" >&2
|
|
exit 3
|
|
fi
|
|
|
|
pid=$1
|
|
st=$2
|
|
url=$3
|
|
jobs_s=${4:-}
|
|
empty_since=$now
|
|
if [[ "$first_seen" -eq 0 ]]; then
|
|
echo "Watching pipeline $pid status=$st $url" >&2
|
|
first_seen=1
|
|
fi
|
|
|
|
case "$st" in
|
|
success)
|
|
echo "OK pipeline=$url sha=$sha id=$pid jobs=$jobs_s"
|
|
exit 0
|
|
;;
|
|
failed|failure|canceled|cancelled|skipped)
|
|
if [[ "$forge_kind" == gitlab ]]; then
|
|
jobs_s=$(gitlab_jobs "$pid" | summarize_jobs || true)
|
|
fi
|
|
echo "FAIL reason=job_$st sha=$sha pipeline=$url jobs=$jobs_s" >&2
|
|
exit 1
|
|
;;
|
|
created|pending|running|waiting_for_resource|preparing|waiting|queued|in_progress|requested)
|
|
sleep "$poll_sec"
|
|
;;
|
|
*)
|
|
sleep "$poll_sec"
|
|
;;
|
|
esac
|
|
done
|