init: resume 작성 하네스 설계
This commit is contained in:
DongHyeonka
@@ -0,0 +1,245 @@
|
||||
"""Trusted quality metrics derived from canonical resume artifacts."""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass
|
||||
import hashlib
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from .models import (
|
||||
CandidateProfile,
|
||||
ContentPlan,
|
||||
EvidenceMap,
|
||||
EvidenceMatchType,
|
||||
GenerationConfig,
|
||||
JobAnalysis,
|
||||
JobPosting,
|
||||
QualityCategory,
|
||||
QualityFinding,
|
||||
QualitySeverity,
|
||||
RequirementKind,
|
||||
ResumeDraft,
|
||||
_claim_mentions_requirement,
|
||||
_evidence_supports_requirement,
|
||||
)
|
||||
|
||||
|
||||
RUBRIC_WEIGHTS: dict[QualityCategory, int] = {
|
||||
QualityCategory.EVIDENCE: 25,
|
||||
QualityCategory.JOB_ALIGNMENT: 20,
|
||||
QualityCategory.COMPLETENESS: 15,
|
||||
QualityCategory.KOREAN_LANGUAGE: 15,
|
||||
QualityCategory.READABILITY: 10,
|
||||
QualityCategory.FORMATTING: 5,
|
||||
QualityCategory.CONSISTENCY: 5,
|
||||
QualityCategory.PRIVACY: 5,
|
||||
}
|
||||
|
||||
QUALITY_POLICY_VERSION = "1.1.0"
|
||||
DETERMINISTIC_BLOCKING_SCORE_CAP = 59.0
|
||||
DETERMINISTIC_WARNING_SCORE_CAP = 89.0
|
||||
|
||||
|
||||
def compute_evaluation_policy_fingerprint(
|
||||
*,
|
||||
system_prompt: str | None = None,
|
||||
evaluator_prompt: str | None = None,
|
||||
) -> str:
|
||||
"""Fingerprint the independent judge policy used to approve a resume.
|
||||
|
||||
Custom prompt repositories must supply their exact prompt text. Callers
|
||||
that omit it are bound to the package-local, wheel-distributed templates.
|
||||
"""
|
||||
|
||||
prompt_root = Path(__file__).resolve().with_name("prompt_templates")
|
||||
if system_prompt is None:
|
||||
system_prompt = (prompt_root / "base-system.md").read_text(
|
||||
encoding="utf-8"
|
||||
).strip()
|
||||
if evaluator_prompt is None:
|
||||
evaluator_prompt = (prompt_root / "evaluate-resume.md").read_text(
|
||||
encoding="utf-8"
|
||||
).strip()
|
||||
payload = {
|
||||
"policy_version": QUALITY_POLICY_VERSION,
|
||||
"system_prompt": system_prompt,
|
||||
"evaluator_prompt": evaluator_prompt,
|
||||
"rubric_weights": {
|
||||
category.value: weight for category, weight in RUBRIC_WEIGHTS.items()
|
||||
},
|
||||
}
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(canonical).hexdigest()
|
||||
|
||||
|
||||
@dataclass(frozen=True, slots=True)
|
||||
class CoverageMetrics:
|
||||
"""Coverage values that must not be supplied by the evaluating model."""
|
||||
|
||||
evidence: float
|
||||
requirements: float
|
||||
|
||||
|
||||
def compute_evaluation_fingerprint(
|
||||
profile: CandidateProfile,
|
||||
draft: ResumeDraft,
|
||||
posting: JobPosting,
|
||||
analysis: JobAnalysis,
|
||||
evidence_map: EvidenceMap,
|
||||
plan: ContentPlan,
|
||||
config: GenerationConfig,
|
||||
*,
|
||||
policy_fingerprint: str | None = None,
|
||||
) -> str:
|
||||
"""Bind a quality decision to every artifact that can change release gates."""
|
||||
|
||||
payload = {
|
||||
"candidate_profile": profile.model_dump(mode="json", exclude={"updated_at"}),
|
||||
"draft": draft.model_dump(mode="json", exclude={"generated_at"}),
|
||||
"posting": posting.model_dump(mode="json", exclude={"collected_at"}),
|
||||
"analysis": analysis.model_dump(mode="json", exclude={"analysed_at"}),
|
||||
"evidence_map": evidence_map.model_dump(
|
||||
mode="json", exclude={"generated_at"}
|
||||
),
|
||||
"content_plan": plan.model_dump(mode="json", exclude={"created_at"}),
|
||||
"config": config.model_dump(mode="json"),
|
||||
"evaluation_policy_fingerprint": (
|
||||
policy_fingerprint or compute_evaluation_policy_fingerprint()
|
||||
),
|
||||
}
|
||||
canonical = json.dumps(
|
||||
payload,
|
||||
ensure_ascii=False,
|
||||
sort_keys=True,
|
||||
separators=(",", ":"),
|
||||
).encode("utf-8")
|
||||
return hashlib.sha256(canonical).hexdigest()
|
||||
|
||||
|
||||
def compute_weighted_overall(
|
||||
category_scores: dict[QualityCategory, float],
|
||||
) -> float:
|
||||
"""Compute the documented 100-point rubric from required category scores."""
|
||||
|
||||
missing = [
|
||||
category.value
|
||||
for category in RUBRIC_WEIGHTS
|
||||
if category not in category_scores
|
||||
]
|
||||
if missing:
|
||||
raise ValueError(f"missing weighted rubric categories: {missing}")
|
||||
weighted = sum(
|
||||
category_scores[category] * weight
|
||||
for category, weight in RUBRIC_WEIGHTS.items()
|
||||
) / 100
|
||||
return round(weighted, 2)
|
||||
|
||||
|
||||
def apply_deterministic_score_caps(
|
||||
category_scores: dict[QualityCategory, float],
|
||||
findings: list[QualityFinding] | tuple[QualityFinding, ...],
|
||||
) -> dict[QualityCategory, float]:
|
||||
"""Prevent subjective scores from contradicting deterministic defects."""
|
||||
|
||||
capped = dict(category_scores)
|
||||
for finding in findings:
|
||||
if finding.category not in RUBRIC_WEIGHTS:
|
||||
continue
|
||||
if finding.blocking:
|
||||
cap = DETERMINISTIC_BLOCKING_SCORE_CAP
|
||||
elif finding.severity is QualitySeverity.WARNING:
|
||||
cap = DETERMINISTIC_WARNING_SCORE_CAP
|
||||
else:
|
||||
continue
|
||||
if finding.category in capped:
|
||||
capped[finding.category] = min(capped[finding.category], cap)
|
||||
return capped
|
||||
|
||||
|
||||
def compute_coverage(
|
||||
draft: ResumeDraft,
|
||||
analysis: JobAnalysis,
|
||||
evidence_map: EvidenceMap,
|
||||
profile: CandidateProfile,
|
||||
) -> CoverageMetrics:
|
||||
"""Compute claim evidence and priority-weighted job-requirement coverage.
|
||||
|
||||
Context-only posting notes are excluded. Required, preferred, and
|
||||
responsibility items remain in the denominator so a high judge score cannot
|
||||
hide an omitted job criterion. Cross-model reference validation is expected
|
||||
to run before this function.
|
||||
"""
|
||||
|
||||
claims = [claim for section in draft.sections for claim in section.claims]
|
||||
evidence_coverage = (
|
||||
sum(bool(claim.evidence_ids) for claim in claims) / len(claims)
|
||||
if claims
|
||||
else 0.0
|
||||
)
|
||||
|
||||
scored_requirements = [
|
||||
requirement
|
||||
for requirement in analysis.requirements
|
||||
if requirement.kind is not RequirementKind.CONTEXT
|
||||
]
|
||||
if not scored_requirements:
|
||||
requirement_coverage = 1.0
|
||||
else:
|
||||
matches = {match.requirement_id: match for match in evidence_map.matches}
|
||||
requirements_by_id = {
|
||||
requirement.requirement_id: requirement
|
||||
for requirement in scored_requirements
|
||||
}
|
||||
covered_ids: set[str] = set()
|
||||
for claim in claims:
|
||||
claim_evidence = set(claim.evidence_ids)
|
||||
for requirement_id in claim.requirement_ids:
|
||||
match = matches.get(requirement_id)
|
||||
requirement = requirements_by_id.get(requirement_id)
|
||||
if (
|
||||
match is not None
|
||||
and requirement is not None
|
||||
and match.match_type is not EvidenceMatchType.GAP
|
||||
and bool(claim_evidence & set(match.evidence_ids))
|
||||
and _claim_mentions_requirement(claim.text, requirement)
|
||||
and any(
|
||||
evidence_id in profile.evidence_by_id
|
||||
and _evidence_supports_requirement(
|
||||
profile.evidence_by_id[evidence_id],
|
||||
requirement,
|
||||
direct=match.match_type is EvidenceMatchType.DIRECT,
|
||||
)
|
||||
for evidence_id in claim_evidence & set(match.evidence_ids)
|
||||
)
|
||||
):
|
||||
covered_ids.add(requirement_id)
|
||||
total_weight = sum(requirement.priority for requirement in scored_requirements)
|
||||
covered_weight = sum(
|
||||
requirement.priority
|
||||
for requirement in scored_requirements
|
||||
if requirement.requirement_id in covered_ids
|
||||
)
|
||||
requirement_coverage = covered_weight / total_weight
|
||||
|
||||
return CoverageMetrics(
|
||||
evidence=evidence_coverage,
|
||||
requirements=requirement_coverage,
|
||||
)
|
||||
|
||||
|
||||
__all__ = [
|
||||
"CoverageMetrics",
|
||||
"QUALITY_POLICY_VERSION",
|
||||
"RUBRIC_WEIGHTS",
|
||||
"apply_deterministic_score_caps",
|
||||
"compute_coverage",
|
||||
"compute_evaluation_fingerprint",
|
||||
"compute_evaluation_policy_fingerprint",
|
||||
"compute_weighted_overall",
|
||||
]
|
||||
Reference in New Issue
Block a user