There is no number for how long CI runs wait, how busy the one runner is, how many runs a pull request costs or how long main stays red. Run times were added up by hand from single runs, so whether a change helped could not be told apart from load. infrastructure/ci/metrics/ci_metrics.py reads the Gitea Actions API (runs, their jobs, closed pull requests) for the repositories the runner serves and prints the CI manual's metrics 1-5 per repository and stage: run execution p50/p90, queue wait p50/p90, runner utilisation per KST day, pull request runs per merged pull request, and main red time. It only reads; credentials come from GITEA_TOKEN or `git credential fill` and are never printed. The arithmetic follows what our Gitea 1.27 returns, which differs from the API reference: a run has no created_at (the earliest job created_at is used), unstarted runs and jobs report 1970-01-01 times (treated as absent), and a job's created_at is its run's, so queue wait is measured per run (first job start minus run creation) instead of per job. Verification: python3 -m unittest test_ci_metrics (11 offline tests) and one run against the server for 2026-11-07..13 (two repositories, 30 s, rc=0). The sensitive-source scan needs rg and ran on the desktop host. Refs: design-docs ci/03-time-reduction-plan.md M-0
385 lines
15 KiB
Python
385 lines
15 KiB
Python
#!/usr/bin/env python3
|
|
"""CI metrics for the repositories one Gitea Actions runner serves.
|
|
|
|
Reads the Gitea Actions API (runs, their jobs, closed pull requests) and prints, per repository and
|
|
stage, the CI manual's metrics 1-5: run execution time, queue wait, runner utilisation, pull
|
|
request runs per merged pull request and time main stayed red. Read-only; it never writes to the
|
|
server.
|
|
|
|
What our Gitea 1.27 returns differs from the gitea.com API reference in three places, and the
|
|
arithmetic below follows the server:
|
|
|
|
* A run carries no ``created_at``. A run's jobs are all created with the run, so the earliest job
|
|
``created_at`` is the run's creation time.
|
|
* A run or job that has not started reports ``started_at`` and ``completed_at`` as
|
|
``1970-01-01T00:00:00Z``. Those are treated as absent.
|
|
* A job's ``created_at`` is the run's, not the moment it became runnable. One runner runs a run's
|
|
jobs one after another, so ``started_at - created_at`` of a later job counts the earlier jobs'
|
|
execution as waiting. Queue wait is therefore measured per run: first job start minus run
|
|
creation.
|
|
|
|
Credentials: ``GITEA_TOKEN`` (a read-only token) if set, otherwise ``git credential fill`` for the
|
|
server host. Neither is printed.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import argparse
|
|
import base64
|
|
import json
|
|
import math
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
import urllib.parse
|
|
import urllib.request
|
|
from dataclasses import dataclass, field
|
|
from datetime import date, datetime, timedelta, timezone
|
|
|
|
EPOCH_PREFIXES = ("1970-01-01", "0001-01-01")
|
|
FINISHED = {"success", "failure"}
|
|
|
|
|
|
def parse_time(value: str | None) -> datetime | None:
|
|
"""An API timestamp, or None for a missing or zero value."""
|
|
if not value or value.startswith(EPOCH_PREFIXES):
|
|
return None
|
|
return datetime.fromisoformat(value.replace("Z", "+00:00"))
|
|
|
|
|
|
def percentile(values: list[float], fraction: float) -> float | None:
|
|
"""Nearest-rank percentile; None for no values."""
|
|
if not values:
|
|
return None
|
|
ordered = sorted(values)
|
|
rank = max(1, math.ceil(fraction * len(ordered)))
|
|
return ordered[rank - 1]
|
|
|
|
|
|
@dataclass
|
|
class Job:
|
|
name: str
|
|
created: datetime | None
|
|
started: datetime | None
|
|
completed: datetime | None
|
|
conclusion: str | None
|
|
|
|
@property
|
|
def minutes(self) -> float:
|
|
if self.started is None or self.completed is None:
|
|
return 0.0
|
|
return max(0.0, (self.completed - self.started).total_seconds() / 60)
|
|
|
|
|
|
@dataclass
|
|
class Run:
|
|
repo: str
|
|
id: int
|
|
workflow: str
|
|
ref: str
|
|
event: str
|
|
status: str
|
|
conclusion: str | None
|
|
pull_request: int | None
|
|
jobs: list[Job] = field(default_factory=list)
|
|
|
|
@property
|
|
def stage(self) -> str:
|
|
return f"{self.workflow}:{self.event}"
|
|
|
|
@property
|
|
def created(self) -> datetime | None:
|
|
times = [job.created for job in self.jobs if job.created is not None]
|
|
return min(times) if times else None
|
|
|
|
@property
|
|
def first_start(self) -> datetime | None:
|
|
times = [job.started for job in self.jobs if job.started is not None]
|
|
return min(times) if times else None
|
|
|
|
@property
|
|
def completed(self) -> datetime | None:
|
|
times = [job.completed for job in self.jobs if job.completed is not None]
|
|
return max(times) if times and self.status == "completed" else None
|
|
|
|
@property
|
|
def execution_minutes(self) -> float:
|
|
return sum(job.minutes for job in self.jobs)
|
|
|
|
@property
|
|
def queue_minutes(self) -> float | None:
|
|
if self.created is None or self.first_start is None:
|
|
return None
|
|
return max(0.0, (self.first_start - self.created).total_seconds() / 60)
|
|
|
|
@property
|
|
def main_push(self) -> bool:
|
|
return self.event == "push" and self.ref == "refs/heads/main"
|
|
|
|
|
|
def run_from_api(repo: str, raw: dict, jobs: list[dict]) -> Run:
|
|
workflow, _, ref = raw.get("path", "").partition("@")
|
|
pulls = raw.get("pull_requests") or []
|
|
return Run(
|
|
repo=repo,
|
|
id=raw["id"],
|
|
workflow=workflow,
|
|
ref=ref,
|
|
event=raw.get("event", ""),
|
|
status=raw.get("status", ""),
|
|
conclusion=raw.get("conclusion"),
|
|
pull_request=pulls[0]["number"] if pulls else None,
|
|
jobs=[
|
|
Job(
|
|
name=job.get("name", ""),
|
|
created=parse_time(job.get("created_at")),
|
|
started=parse_time(job.get("started_at")),
|
|
completed=parse_time(job.get("completed_at")),
|
|
conclusion=job.get("conclusion"),
|
|
)
|
|
for job in jobs
|
|
],
|
|
)
|
|
|
|
|
|
class Gitea:
|
|
"""The few read-only API calls this script makes."""
|
|
|
|
def __init__(self, server: str):
|
|
self.server = server.rstrip("/")
|
|
self.header = self._authorization()
|
|
|
|
def _authorization(self) -> str:
|
|
token = os.environ.get("GITEA_TOKEN")
|
|
if token:
|
|
return f"token {token}"
|
|
host = urllib.parse.urlparse(self.server).netloc
|
|
answer = subprocess.run(
|
|
["git", "credential", "fill"],
|
|
input=f"protocol=https\nhost={host}\n\n",
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
).stdout
|
|
fields = dict(line.split("=", 1) for line in answer.splitlines() if "=" in line)
|
|
pair = f"{fields['username']}:{fields['password']}".encode()
|
|
return "Basic " + base64.b64encode(pair).decode()
|
|
|
|
def get(self, path: str) -> object:
|
|
request = urllib.request.Request(
|
|
f"{self.server}/api/v1{path}",
|
|
headers={"Authorization": self.header, "Accept": "application/json"},
|
|
)
|
|
with urllib.request.urlopen(request, timeout=30) as response:
|
|
return json.load(response)
|
|
|
|
|
|
def fetch_runs(api: Gitea, owner: str, repo: str, since: datetime) -> list[Run]:
|
|
"""Runs created at or after ``since``, newest first. Run ids grow with creation time."""
|
|
runs: list[Run] = []
|
|
page = 1
|
|
while True:
|
|
body = api.get(f"/repos/{owner}/{repo}/actions/runs?limit=50&page={page}")
|
|
batch = body.get("workflow_runs") or []
|
|
if not batch:
|
|
return runs
|
|
for raw in batch:
|
|
jobs = api.get(f"/repos/{owner}/{repo}/actions/runs/{raw['id']}/jobs").get("jobs") or []
|
|
run = run_from_api(repo, raw, jobs)
|
|
if run.created is not None and run.created < since:
|
|
return runs
|
|
runs.append(run)
|
|
page += 1
|
|
|
|
|
|
def fetch_merged_pulls(api: Gitea, owner: str, repo: str, since: datetime, until: datetime) -> list[int]:
|
|
numbers: list[int] = []
|
|
page = 1
|
|
while True:
|
|
batch = api.get(f"/repos/{owner}/{repo}/pulls?state=closed&sort=recentupdate&limit=50&page={page}")
|
|
if not batch:
|
|
return numbers
|
|
for pull in batch:
|
|
merged = parse_time(pull.get("merged_at"))
|
|
if pull.get("merged") and merged is not None and since <= merged < until:
|
|
numbers.append(pull["number"])
|
|
updated = [parse_time(pull.get("updated_at")) for pull in batch]
|
|
if all(time is not None and time < since for time in updated):
|
|
return numbers
|
|
page += 1
|
|
|
|
|
|
@dataclass
|
|
class RedEpisode:
|
|
workflow: str
|
|
first_failed_run: int
|
|
fixed_by_run: int | None
|
|
hours: float
|
|
|
|
|
|
def red_episodes(runs: list[Run], now: datetime) -> list[RedEpisode]:
|
|
"""Main stays red from the first failing main push run's creation to the next green one's end."""
|
|
episodes: list[RedEpisode] = []
|
|
by_workflow: dict[str, list[Run]] = {}
|
|
for run in runs:
|
|
if run.main_push and run.status == "completed" and run.conclusion in FINISHED:
|
|
by_workflow.setdefault(run.workflow, []).append(run)
|
|
for workflow, history in by_workflow.items():
|
|
history.sort(key=lambda run: run.created or now)
|
|
start: Run | None = None
|
|
for run in history:
|
|
if run.conclusion == "failure" and start is None:
|
|
start = run
|
|
elif run.conclusion == "success" and start is not None:
|
|
end = run.completed or now
|
|
episodes.append(
|
|
RedEpisode(workflow, start.id, run.id, (end - start.created).total_seconds() / 3600)
|
|
)
|
|
start = None
|
|
if start is not None:
|
|
episodes.append(RedEpisode(workflow, start.id, None, (now - start.created).total_seconds() / 3600))
|
|
return episodes
|
|
|
|
|
|
def stage_rows(runs: list[Run]) -> list[dict]:
|
|
rows = []
|
|
for stage in sorted({run.stage for run in runs}):
|
|
finished = [run for run in runs if run.stage == stage and run.conclusion in FINISHED]
|
|
waited = [run.queue_minutes for run in runs if run.stage == stage and run.queue_minutes is not None]
|
|
executed = [run.execution_minutes for run in finished]
|
|
rows.append(
|
|
{
|
|
"stage": stage,
|
|
"runs": len([run for run in runs if run.stage == stage]),
|
|
"finished": len(finished),
|
|
"red": len([run for run in finished if run.conclusion == "failure"]),
|
|
"exec_p50": percentile(executed, 0.5),
|
|
"exec_p90": percentile(executed, 0.9),
|
|
"wait_p50": percentile(waited, 0.5),
|
|
"wait_p90": percentile(waited, 0.9),
|
|
}
|
|
)
|
|
return rows
|
|
|
|
|
|
def utilisation(runs: list[Run], offset: timedelta) -> dict[date, float]:
|
|
"""Job execution minutes per local day (job start day) across every repository given."""
|
|
minutes: dict[date, float] = {}
|
|
for run in runs:
|
|
for job in run.jobs:
|
|
if job.started is not None and job.minutes > 0:
|
|
day = (job.started + offset).date()
|
|
minutes[day] = minutes.get(day, 0.0) + job.minutes
|
|
return minutes
|
|
|
|
|
|
def runs_per_merged_pull(runs: list[Run], merged: list[int]) -> list[int]:
|
|
return [len([run for run in runs if run.pull_request == number]) for number in merged]
|
|
|
|
|
|
def fmt(value: float | None, digits: int = 1) -> str:
|
|
return "-" if value is None else f"{value:.{digits}f}"
|
|
|
|
|
|
def report(window: str, per_repo: dict[str, dict], days: dict[date, float]) -> str:
|
|
lines = [f"# CI 지표 ({window})", ""]
|
|
for repo, data in per_repo.items():
|
|
lines += [
|
|
f"## {repo}",
|
|
"",
|
|
"| 단계 | run | 끝난 run | 빨강 | 실행 p50(분) | 실행 p90(분) | 대기 p50(분) | 대기 p90(분) |",
|
|
"|---|---|---|---|---|---|---|---|",
|
|
]
|
|
for row in data["stages"]:
|
|
lines.append(
|
|
f"| {row['stage']} | {row['runs']} | {row['finished']} | {row['red']} | {fmt(row['exec_p50'])} | "
|
|
f"{fmt(row['exec_p90'])} | {fmt(row['wait_p50'])} | {fmt(row['wait_p90'])} |"
|
|
)
|
|
counts = data["runs_per_pull"]
|
|
lines += [
|
|
"",
|
|
f"- 병합된 PR {len(counts)}개, PR당 run 수 p50 {fmt(percentile(counts, 0.5), 0)}, "
|
|
f"p90 {fmt(percentile(counts, 0.9), 0)}, 최대 {max(counts) if counts else '-'}",
|
|
]
|
|
episodes = data["red"]
|
|
if episodes:
|
|
longest = max(episodes, key=lambda episode: episode.hours)
|
|
lines.append(
|
|
f"- main 빨강 {len(episodes)}회, 합 {sum(e.hours for e in episodes):.1f}시간, p90 "
|
|
f"{fmt(percentile([e.hours for e in episodes], 0.9))}시간, 가장 긴 것 {longest.hours:.1f}시간"
|
|
f"({longest.workflow} run {longest.first_failed_run} → "
|
|
f"{longest.fixed_by_run if longest.fixed_by_run else '아직 빨강'})"
|
|
)
|
|
else:
|
|
lines.append("- main 빨강 0회")
|
|
lines.append("")
|
|
lines += ["## runner 사용률 (위 repo 합, job 시작 날짜 기준)", "", "| 날짜 | 실행 합(분) | 사용률 |", "|---|---|---|"]
|
|
for day in sorted(days):
|
|
lines.append(f"| {day} | {days[day]:.0f} | {days[day] / 1440:.0%} |")
|
|
if days:
|
|
busiest = max(days, key=days.get)
|
|
lines += ["", f"- 바쁜 날 {busiest}: {days[busiest] / 1440:.0%}"]
|
|
return "\n".join(lines) + "\n"
|
|
|
|
|
|
def main(argv: list[str]) -> int:
|
|
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
|
|
parser.add_argument("--server", default="https://git.learn.hyeonworks.com")
|
|
parser.add_argument("--owner", default="donghyeon.kang")
|
|
parser.add_argument("--repo", action="append", required=True, help="repeat for each repository")
|
|
parser.add_argument("--since", required=True, type=date.fromisoformat, help="first local day, YYYY-MM-DD")
|
|
parser.add_argument("--until", required=True, type=date.fromisoformat, help="last local day, inclusive")
|
|
parser.add_argument("--utc-offset-hours", type=int, default=9, help="local day boundary (default KST)")
|
|
parser.add_argument("--json", action="store_true", help="print the numbers as JSON instead of tables")
|
|
args = parser.parse_args(argv)
|
|
|
|
offset = timedelta(hours=args.utc_offset_hours)
|
|
since = datetime.combine(args.since, datetime.min.time(), timezone.utc) - offset
|
|
until = datetime.combine(args.until + timedelta(days=1), datetime.min.time(), timezone.utc) - offset
|
|
# A pull request merged in the window may have had runs in the week before it.
|
|
lookback = since - timedelta(days=7)
|
|
now = min(datetime.now(timezone.utc), until)
|
|
|
|
api = Gitea(args.server)
|
|
per_repo: dict[str, dict] = {}
|
|
window_runs: list[Run] = []
|
|
for repo in args.repo:
|
|
fetched = fetch_runs(api, args.owner, repo, lookback)
|
|
in_window = [run for run in fetched if run.created is not None and since <= run.created < until]
|
|
window_runs += in_window
|
|
merged = fetch_merged_pulls(api, args.owner, repo, since, until)
|
|
per_repo[repo] = {
|
|
"stages": stage_rows(in_window),
|
|
"runs_per_pull": runs_per_merged_pull(fetched, merged),
|
|
"red": red_episodes(in_window, now),
|
|
}
|
|
days = utilisation(window_runs, offset)
|
|
|
|
window = f"{args.since} ~ {args.until}, UTC{args.utc_offset_hours:+d}"
|
|
if args.json:
|
|
print(
|
|
json.dumps(
|
|
{
|
|
"window": window,
|
|
"repos": {
|
|
repo: {
|
|
"stages": data["stages"],
|
|
"runs_per_pull": data["runs_per_pull"],
|
|
"red": [episode.__dict__ for episode in data["red"]],
|
|
}
|
|
for repo, data in per_repo.items()
|
|
},
|
|
"utilisation_minutes": {str(day): round(value, 1) for day, value in sorted(days.items())},
|
|
},
|
|
ensure_ascii=False,
|
|
indent=2,
|
|
)
|
|
)
|
|
else:
|
|
print(report(window, per_repo, days), end="")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main(sys.argv[1:]))
|