246 lines
7.9 KiB
Python
246 lines
7.9 KiB
Python
"""Trusted quality metrics derived from canonical resume artifacts."""
|
|
|
|
from __future__ import annotations
|
|
|
|
from dataclasses import dataclass
|
|
import hashlib
|
|
import json
|
|
from pathlib import Path
|
|
|
|
from .models import (
|
|
CandidateProfile,
|
|
ContentPlan,
|
|
EvidenceMap,
|
|
EvidenceMatchType,
|
|
GenerationConfig,
|
|
JobAnalysis,
|
|
JobPosting,
|
|
QualityCategory,
|
|
QualityFinding,
|
|
QualitySeverity,
|
|
RequirementKind,
|
|
ResumeDraft,
|
|
_claim_mentions_requirement,
|
|
_evidence_supports_requirement,
|
|
)
|
|
|
|
|
|
RUBRIC_WEIGHTS: dict[QualityCategory, int] = {
|
|
QualityCategory.EVIDENCE: 25,
|
|
QualityCategory.JOB_ALIGNMENT: 20,
|
|
QualityCategory.COMPLETENESS: 15,
|
|
QualityCategory.KOREAN_LANGUAGE: 15,
|
|
QualityCategory.READABILITY: 10,
|
|
QualityCategory.FORMATTING: 5,
|
|
QualityCategory.CONSISTENCY: 5,
|
|
QualityCategory.PRIVACY: 5,
|
|
}
|
|
|
|
QUALITY_POLICY_VERSION = "1.1.0"
|
|
DETERMINISTIC_BLOCKING_SCORE_CAP = 59.0
|
|
DETERMINISTIC_WARNING_SCORE_CAP = 89.0
|
|
|
|
|
|
def compute_evaluation_policy_fingerprint(
|
|
*,
|
|
system_prompt: str | None = None,
|
|
evaluator_prompt: str | None = None,
|
|
) -> str:
|
|
"""Fingerprint the independent judge policy used to approve a resume.
|
|
|
|
Custom prompt repositories must supply their exact prompt text. Callers
|
|
that omit it are bound to the package-local, wheel-distributed templates.
|
|
"""
|
|
|
|
prompt_root = Path(__file__).resolve().with_name("prompt_templates")
|
|
if system_prompt is None:
|
|
system_prompt = (prompt_root / "base-system.md").read_text(
|
|
encoding="utf-8"
|
|
).strip()
|
|
if evaluator_prompt is None:
|
|
evaluator_prompt = (prompt_root / "evaluate-resume.md").read_text(
|
|
encoding="utf-8"
|
|
).strip()
|
|
payload = {
|
|
"policy_version": QUALITY_POLICY_VERSION,
|
|
"system_prompt": system_prompt,
|
|
"evaluator_prompt": evaluator_prompt,
|
|
"rubric_weights": {
|
|
category.value: weight for category, weight in RUBRIC_WEIGHTS.items()
|
|
},
|
|
}
|
|
canonical = json.dumps(
|
|
payload,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
return hashlib.sha256(canonical).hexdigest()
|
|
|
|
|
|
@dataclass(frozen=True, slots=True)
|
|
class CoverageMetrics:
|
|
"""Coverage values that must not be supplied by the evaluating model."""
|
|
|
|
evidence: float
|
|
requirements: float
|
|
|
|
|
|
def compute_evaluation_fingerprint(
|
|
profile: CandidateProfile,
|
|
draft: ResumeDraft,
|
|
posting: JobPosting,
|
|
analysis: JobAnalysis,
|
|
evidence_map: EvidenceMap,
|
|
plan: ContentPlan,
|
|
config: GenerationConfig,
|
|
*,
|
|
policy_fingerprint: str | None = None,
|
|
) -> str:
|
|
"""Bind a quality decision to every artifact that can change release gates."""
|
|
|
|
payload = {
|
|
"candidate_profile": profile.model_dump(mode="json", exclude={"updated_at"}),
|
|
"draft": draft.model_dump(mode="json", exclude={"generated_at"}),
|
|
"posting": posting.model_dump(mode="json", exclude={"collected_at"}),
|
|
"analysis": analysis.model_dump(mode="json", exclude={"analysed_at"}),
|
|
"evidence_map": evidence_map.model_dump(
|
|
mode="json", exclude={"generated_at"}
|
|
),
|
|
"content_plan": plan.model_dump(mode="json", exclude={"created_at"}),
|
|
"config": config.model_dump(mode="json"),
|
|
"evaluation_policy_fingerprint": (
|
|
policy_fingerprint or compute_evaluation_policy_fingerprint()
|
|
),
|
|
}
|
|
canonical = json.dumps(
|
|
payload,
|
|
ensure_ascii=False,
|
|
sort_keys=True,
|
|
separators=(",", ":"),
|
|
).encode("utf-8")
|
|
return hashlib.sha256(canonical).hexdigest()
|
|
|
|
|
|
def compute_weighted_overall(
|
|
category_scores: dict[QualityCategory, float],
|
|
) -> float:
|
|
"""Compute the documented 100-point rubric from required category scores."""
|
|
|
|
missing = [
|
|
category.value
|
|
for category in RUBRIC_WEIGHTS
|
|
if category not in category_scores
|
|
]
|
|
if missing:
|
|
raise ValueError(f"missing weighted rubric categories: {missing}")
|
|
weighted = sum(
|
|
category_scores[category] * weight
|
|
for category, weight in RUBRIC_WEIGHTS.items()
|
|
) / 100
|
|
return round(weighted, 2)
|
|
|
|
|
|
def apply_deterministic_score_caps(
|
|
category_scores: dict[QualityCategory, float],
|
|
findings: list[QualityFinding] | tuple[QualityFinding, ...],
|
|
) -> dict[QualityCategory, float]:
|
|
"""Prevent subjective scores from contradicting deterministic defects."""
|
|
|
|
capped = dict(category_scores)
|
|
for finding in findings:
|
|
if finding.category not in RUBRIC_WEIGHTS:
|
|
continue
|
|
if finding.blocking:
|
|
cap = DETERMINISTIC_BLOCKING_SCORE_CAP
|
|
elif finding.severity is QualitySeverity.WARNING:
|
|
cap = DETERMINISTIC_WARNING_SCORE_CAP
|
|
else:
|
|
continue
|
|
if finding.category in capped:
|
|
capped[finding.category] = min(capped[finding.category], cap)
|
|
return capped
|
|
|
|
|
|
def compute_coverage(
|
|
draft: ResumeDraft,
|
|
analysis: JobAnalysis,
|
|
evidence_map: EvidenceMap,
|
|
profile: CandidateProfile,
|
|
) -> CoverageMetrics:
|
|
"""Compute claim evidence and priority-weighted job-requirement coverage.
|
|
|
|
Context-only posting notes are excluded. Required, preferred, and
|
|
responsibility items remain in the denominator so a high judge score cannot
|
|
hide an omitted job criterion. Cross-model reference validation is expected
|
|
to run before this function.
|
|
"""
|
|
|
|
claims = [claim for section in draft.sections for claim in section.claims]
|
|
evidence_coverage = (
|
|
sum(bool(claim.evidence_ids) for claim in claims) / len(claims)
|
|
if claims
|
|
else 0.0
|
|
)
|
|
|
|
scored_requirements = [
|
|
requirement
|
|
for requirement in analysis.requirements
|
|
if requirement.kind is not RequirementKind.CONTEXT
|
|
]
|
|
if not scored_requirements:
|
|
requirement_coverage = 1.0
|
|
else:
|
|
matches = {match.requirement_id: match for match in evidence_map.matches}
|
|
requirements_by_id = {
|
|
requirement.requirement_id: requirement
|
|
for requirement in scored_requirements
|
|
}
|
|
covered_ids: set[str] = set()
|
|
for claim in claims:
|
|
claim_evidence = set(claim.evidence_ids)
|
|
for requirement_id in claim.requirement_ids:
|
|
match = matches.get(requirement_id)
|
|
requirement = requirements_by_id.get(requirement_id)
|
|
if (
|
|
match is not None
|
|
and requirement is not None
|
|
and match.match_type is not EvidenceMatchType.GAP
|
|
and bool(claim_evidence & set(match.evidence_ids))
|
|
and _claim_mentions_requirement(claim.text, requirement)
|
|
and any(
|
|
evidence_id in profile.evidence_by_id
|
|
and _evidence_supports_requirement(
|
|
profile.evidence_by_id[evidence_id],
|
|
requirement,
|
|
direct=match.match_type is EvidenceMatchType.DIRECT,
|
|
)
|
|
for evidence_id in claim_evidence & set(match.evidence_ids)
|
|
)
|
|
):
|
|
covered_ids.add(requirement_id)
|
|
total_weight = sum(requirement.priority for requirement in scored_requirements)
|
|
covered_weight = sum(
|
|
requirement.priority
|
|
for requirement in scored_requirements
|
|
if requirement.requirement_id in covered_ids
|
|
)
|
|
requirement_coverage = covered_weight / total_weight
|
|
|
|
return CoverageMetrics(
|
|
evidence=evidence_coverage,
|
|
requirements=requirement_coverage,
|
|
)
|
|
|
|
|
|
__all__ = [
|
|
"CoverageMetrics",
|
|
"QUALITY_POLICY_VERSION",
|
|
"RUBRIC_WEIGHTS",
|
|
"apply_deterministic_score_caps",
|
|
"compute_coverage",
|
|
"compute_evaluation_fingerprint",
|
|
"compute_evaluation_policy_fingerprint",
|
|
"compute_weighted_overall",
|
|
]
|