Files
DongHyeonka dd8e9140b7 feat(ci): add a read-only CI metrics script for the desktop runner
There is no number for how long CI runs wait, how busy the one runner is,
how many runs a pull request costs or how long main stays red. Run times
were added up by hand from single runs, so whether a change helped could
not be told apart from load.

infrastructure/ci/metrics/ci_metrics.py reads the Gitea Actions API (runs,
their jobs, closed pull requests) for the repositories the runner serves
and prints the CI manual's metrics 1-5 per repository and stage: run
execution p50/p90, queue wait p50/p90, runner utilisation per KST day,
pull request runs per merged pull request, and main red time. It only
reads; credentials come from GITEA_TOKEN or `git credential fill` and are
never printed.

The arithmetic follows what our Gitea 1.27 returns, which differs from the
API reference: a run has no created_at (the earliest job created_at is
used), unstarted runs and jobs report 1970-01-01 times (treated as
absent), and a job's created_at is its run's, so queue wait is measured
per run (first job start minus run creation) instead of per job.

Verification: python3 -m unittest test_ci_metrics (11 offline tests) and
one run against the server for 2026-11-07..13 (two repositories, 30 s,
rc=0). The sensitive-source scan needs rg and ran on the desktop host.

Refs: design-docs ci/03-time-reduction-plan.md M-0
2026-11-13 19:34:38 +09:00

385 lines
15 KiB
Python

#!/usr/bin/env python3
"""CI metrics for the repositories one Gitea Actions runner serves.
Reads the Gitea Actions API (runs, their jobs, closed pull requests) and prints, per repository and
stage, the CI manual's metrics 1-5: run execution time, queue wait, runner utilisation, pull
request runs per merged pull request and time main stayed red. Read-only; it never writes to the
server.
What our Gitea 1.27 returns differs from the gitea.com API reference in three places, and the
arithmetic below follows the server:
* A run carries no ``created_at``. A run's jobs are all created with the run, so the earliest job
``created_at`` is the run's creation time.
* A run or job that has not started reports ``started_at`` and ``completed_at`` as
``1970-01-01T00:00:00Z``. Those are treated as absent.
* A job's ``created_at`` is the run's, not the moment it became runnable. One runner runs a run's
jobs one after another, so ``started_at - created_at`` of a later job counts the earlier jobs'
execution as waiting. Queue wait is therefore measured per run: first job start minus run
creation.
Credentials: ``GITEA_TOKEN`` (a read-only token) if set, otherwise ``git credential fill`` for the
server host. Neither is printed.
"""
from __future__ import annotations
import argparse
import base64
import json
import math
import os
import subprocess
import sys
import urllib.parse
import urllib.request
from dataclasses import dataclass, field
from datetime import date, datetime, timedelta, timezone
EPOCH_PREFIXES = ("1970-01-01", "0001-01-01")
FINISHED = {"success", "failure"}
def parse_time(value: str | None) -> datetime | None:
"""An API timestamp, or None for a missing or zero value."""
if not value or value.startswith(EPOCH_PREFIXES):
return None
return datetime.fromisoformat(value.replace("Z", "+00:00"))
def percentile(values: list[float], fraction: float) -> float | None:
"""Nearest-rank percentile; None for no values."""
if not values:
return None
ordered = sorted(values)
rank = max(1, math.ceil(fraction * len(ordered)))
return ordered[rank - 1]
@dataclass
class Job:
name: str
created: datetime | None
started: datetime | None
completed: datetime | None
conclusion: str | None
@property
def minutes(self) -> float:
if self.started is None or self.completed is None:
return 0.0
return max(0.0, (self.completed - self.started).total_seconds() / 60)
@dataclass
class Run:
repo: str
id: int
workflow: str
ref: str
event: str
status: str
conclusion: str | None
pull_request: int | None
jobs: list[Job] = field(default_factory=list)
@property
def stage(self) -> str:
return f"{self.workflow}:{self.event}"
@property
def created(self) -> datetime | None:
times = [job.created for job in self.jobs if job.created is not None]
return min(times) if times else None
@property
def first_start(self) -> datetime | None:
times = [job.started for job in self.jobs if job.started is not None]
return min(times) if times else None
@property
def completed(self) -> datetime | None:
times = [job.completed for job in self.jobs if job.completed is not None]
return max(times) if times and self.status == "completed" else None
@property
def execution_minutes(self) -> float:
return sum(job.minutes for job in self.jobs)
@property
def queue_minutes(self) -> float | None:
if self.created is None or self.first_start is None:
return None
return max(0.0, (self.first_start - self.created).total_seconds() / 60)
@property
def main_push(self) -> bool:
return self.event == "push" and self.ref == "refs/heads/main"
def run_from_api(repo: str, raw: dict, jobs: list[dict]) -> Run:
workflow, _, ref = raw.get("path", "").partition("@")
pulls = raw.get("pull_requests") or []
return Run(
repo=repo,
id=raw["id"],
workflow=workflow,
ref=ref,
event=raw.get("event", ""),
status=raw.get("status", ""),
conclusion=raw.get("conclusion"),
pull_request=pulls[0]["number"] if pulls else None,
jobs=[
Job(
name=job.get("name", ""),
created=parse_time(job.get("created_at")),
started=parse_time(job.get("started_at")),
completed=parse_time(job.get("completed_at")),
conclusion=job.get("conclusion"),
)
for job in jobs
],
)
class Gitea:
"""The few read-only API calls this script makes."""
def __init__(self, server: str):
self.server = server.rstrip("/")
self.header = self._authorization()
def _authorization(self) -> str:
token = os.environ.get("GITEA_TOKEN")
if token:
return f"token {token}"
host = urllib.parse.urlparse(self.server).netloc
answer = subprocess.run(
["git", "credential", "fill"],
input=f"protocol=https\nhost={host}\n\n",
capture_output=True,
text=True,
check=True,
).stdout
fields = dict(line.split("=", 1) for line in answer.splitlines() if "=" in line)
pair = f"{fields['username']}:{fields['password']}".encode()
return "Basic " + base64.b64encode(pair).decode()
def get(self, path: str) -> object:
request = urllib.request.Request(
f"{self.server}/api/v1{path}",
headers={"Authorization": self.header, "Accept": "application/json"},
)
with urllib.request.urlopen(request, timeout=30) as response:
return json.load(response)
def fetch_runs(api: Gitea, owner: str, repo: str, since: datetime) -> list[Run]:
"""Runs created at or after ``since``, newest first. Run ids grow with creation time."""
runs: list[Run] = []
page = 1
while True:
body = api.get(f"/repos/{owner}/{repo}/actions/runs?limit=50&page={page}")
batch = body.get("workflow_runs") or []
if not batch:
return runs
for raw in batch:
jobs = api.get(f"/repos/{owner}/{repo}/actions/runs/{raw['id']}/jobs").get("jobs") or []
run = run_from_api(repo, raw, jobs)
if run.created is not None and run.created < since:
return runs
runs.append(run)
page += 1
def fetch_merged_pulls(api: Gitea, owner: str, repo: str, since: datetime, until: datetime) -> list[int]:
numbers: list[int] = []
page = 1
while True:
batch = api.get(f"/repos/{owner}/{repo}/pulls?state=closed&sort=recentupdate&limit=50&page={page}")
if not batch:
return numbers
for pull in batch:
merged = parse_time(pull.get("merged_at"))
if pull.get("merged") and merged is not None and since <= merged < until:
numbers.append(pull["number"])
updated = [parse_time(pull.get("updated_at")) for pull in batch]
if all(time is not None and time < since for time in updated):
return numbers
page += 1
@dataclass
class RedEpisode:
workflow: str
first_failed_run: int
fixed_by_run: int | None
hours: float
def red_episodes(runs: list[Run], now: datetime) -> list[RedEpisode]:
"""Main stays red from the first failing main push run's creation to the next green one's end."""
episodes: list[RedEpisode] = []
by_workflow: dict[str, list[Run]] = {}
for run in runs:
if run.main_push and run.status == "completed" and run.conclusion in FINISHED:
by_workflow.setdefault(run.workflow, []).append(run)
for workflow, history in by_workflow.items():
history.sort(key=lambda run: run.created or now)
start: Run | None = None
for run in history:
if run.conclusion == "failure" and start is None:
start = run
elif run.conclusion == "success" and start is not None:
end = run.completed or now
episodes.append(
RedEpisode(workflow, start.id, run.id, (end - start.created).total_seconds() / 3600)
)
start = None
if start is not None:
episodes.append(RedEpisode(workflow, start.id, None, (now - start.created).total_seconds() / 3600))
return episodes
def stage_rows(runs: list[Run]) -> list[dict]:
rows = []
for stage in sorted({run.stage for run in runs}):
finished = [run for run in runs if run.stage == stage and run.conclusion in FINISHED]
waited = [run.queue_minutes for run in runs if run.stage == stage and run.queue_minutes is not None]
executed = [run.execution_minutes for run in finished]
rows.append(
{
"stage": stage,
"runs": len([run for run in runs if run.stage == stage]),
"finished": len(finished),
"red": len([run for run in finished if run.conclusion == "failure"]),
"exec_p50": percentile(executed, 0.5),
"exec_p90": percentile(executed, 0.9),
"wait_p50": percentile(waited, 0.5),
"wait_p90": percentile(waited, 0.9),
}
)
return rows
def utilisation(runs: list[Run], offset: timedelta) -> dict[date, float]:
"""Job execution minutes per local day (job start day) across every repository given."""
minutes: dict[date, float] = {}
for run in runs:
for job in run.jobs:
if job.started is not None and job.minutes > 0:
day = (job.started + offset).date()
minutes[day] = minutes.get(day, 0.0) + job.minutes
return minutes
def runs_per_merged_pull(runs: list[Run], merged: list[int]) -> list[int]:
return [len([run for run in runs if run.pull_request == number]) for number in merged]
def fmt(value: float | None, digits: int = 1) -> str:
return "-" if value is None else f"{value:.{digits}f}"
def report(window: str, per_repo: dict[str, dict], days: dict[date, float]) -> str:
lines = [f"# CI 지표 ({window})", ""]
for repo, data in per_repo.items():
lines += [
f"## {repo}",
"",
"| 단계 | run | 끝난 run | 빨강 | 실행 p50(분) | 실행 p90(분) | 대기 p50(분) | 대기 p90(분) |",
"|---|---|---|---|---|---|---|---|",
]
for row in data["stages"]:
lines.append(
f"| {row['stage']} | {row['runs']} | {row['finished']} | {row['red']} | {fmt(row['exec_p50'])} | "
f"{fmt(row['exec_p90'])} | {fmt(row['wait_p50'])} | {fmt(row['wait_p90'])} |"
)
counts = data["runs_per_pull"]
lines += [
"",
f"- 병합된 PR {len(counts)}개, PR당 run 수 p50 {fmt(percentile(counts, 0.5), 0)}, "
f"p90 {fmt(percentile(counts, 0.9), 0)}, 최대 {max(counts) if counts else '-'}",
]
episodes = data["red"]
if episodes:
longest = max(episodes, key=lambda episode: episode.hours)
lines.append(
f"- main 빨강 {len(episodes)}회, 합 {sum(e.hours for e in episodes):.1f}시간, p90 "
f"{fmt(percentile([e.hours for e in episodes], 0.9))}시간, 가장 긴 것 {longest.hours:.1f}시간"
f"({longest.workflow} run {longest.first_failed_run} → "
f"{longest.fixed_by_run if longest.fixed_by_run else '아직 빨강'})"
)
else:
lines.append("- main 빨강 0회")
lines.append("")
lines += ["## runner 사용률 (위 repo 합, job 시작 날짜 기준)", "", "| 날짜 | 실행 합(분) | 사용률 |", "|---|---|---|"]
for day in sorted(days):
lines.append(f"| {day} | {days[day]:.0f} | {days[day] / 1440:.0%} |")
if days:
busiest = max(days, key=days.get)
lines += ["", f"- 바쁜 날 {busiest}: {days[busiest] / 1440:.0%}"]
return "\n".join(lines) + "\n"
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
parser.add_argument("--server", default="https://git.learn.hyeonworks.com")
parser.add_argument("--owner", default="donghyeon.kang")
parser.add_argument("--repo", action="append", required=True, help="repeat for each repository")
parser.add_argument("--since", required=True, type=date.fromisoformat, help="first local day, YYYY-MM-DD")
parser.add_argument("--until", required=True, type=date.fromisoformat, help="last local day, inclusive")
parser.add_argument("--utc-offset-hours", type=int, default=9, help="local day boundary (default KST)")
parser.add_argument("--json", action="store_true", help="print the numbers as JSON instead of tables")
args = parser.parse_args(argv)
offset = timedelta(hours=args.utc_offset_hours)
since = datetime.combine(args.since, datetime.min.time(), timezone.utc) - offset
until = datetime.combine(args.until + timedelta(days=1), datetime.min.time(), timezone.utc) - offset
# A pull request merged in the window may have had runs in the week before it.
lookback = since - timedelta(days=7)
now = min(datetime.now(timezone.utc), until)
api = Gitea(args.server)
per_repo: dict[str, dict] = {}
window_runs: list[Run] = []
for repo in args.repo:
fetched = fetch_runs(api, args.owner, repo, lookback)
in_window = [run for run in fetched if run.created is not None and since <= run.created < until]
window_runs += in_window
merged = fetch_merged_pulls(api, args.owner, repo, since, until)
per_repo[repo] = {
"stages": stage_rows(in_window),
"runs_per_pull": runs_per_merged_pull(fetched, merged),
"red": red_episodes(in_window, now),
}
days = utilisation(window_runs, offset)
window = f"{args.since} ~ {args.until}, UTC{args.utc_offset_hours:+d}"
if args.json:
print(
json.dumps(
{
"window": window,
"repos": {
repo: {
"stages": data["stages"],
"runs_per_pull": data["runs_per_pull"],
"red": [episode.__dict__ for episode in data["red"]],
}
for repo, data in per_repo.items()
},
"utilisation_minutes": {str(day): round(value, 1) for day, value in sorted(days.items())},
},
ensure_ascii=False,
indent=2,
)
)
else:
print(report(window, per_repo, days), end="")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))