342 lines
10 KiB
Python
342 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
from datetime import datetime, timezone
|
|
|
|
import pytest
|
|
|
|
from resume_harness.models import (
|
|
CandidateProfile,
|
|
ContactInfo,
|
|
DraftClaim,
|
|
DraftSection,
|
|
EvidenceCategory,
|
|
EvidenceItem,
|
|
EvidenceMap,
|
|
EvidenceMatch,
|
|
EvidenceMatchType,
|
|
EvidenceSource,
|
|
JobAnalysis,
|
|
JobRequirement,
|
|
QualityCategory,
|
|
QualityFinding,
|
|
QualitySeverity,
|
|
RequirementCategory,
|
|
RequirementKind,
|
|
ResumeDraft,
|
|
SectionType,
|
|
)
|
|
from resume_harness.quality import (
|
|
apply_deterministic_score_caps,
|
|
compute_coverage,
|
|
compute_evaluation_policy_fingerprint,
|
|
compute_weighted_overall,
|
|
)
|
|
|
|
|
|
NOW = datetime(2026, 7, 1, 12, tzinfo=timezone.utc)
|
|
|
|
|
|
def test_deterministic_blocker_caps_inflated_subjective_score() -> None:
|
|
scores = {
|
|
QualityCategory.EVIDENCE: 100,
|
|
QualityCategory.JOB_ALIGNMENT: 95,
|
|
QualityCategory.COMPLETENESS: 99,
|
|
QualityCategory.KOREAN_LANGUAGE: 95,
|
|
QualityCategory.READABILITY: 95,
|
|
QualityCategory.FORMATTING: 95,
|
|
QualityCategory.CONSISTENCY: 95,
|
|
QualityCategory.PRIVACY: 100,
|
|
}
|
|
blocker = QualityFinding(
|
|
finding_id="thin-summary",
|
|
code="CONTENT.THIN_SUMMARY",
|
|
severity=QualitySeverity.ERROR,
|
|
category=QualityCategory.COMPLETENESS,
|
|
message="핵심 요약이 지나치게 얇다.",
|
|
)
|
|
|
|
capped = apply_deterministic_score_caps(scores, [blocker])
|
|
|
|
assert capped[QualityCategory.COMPLETENESS] == 59
|
|
assert capped[QualityCategory.EVIDENCE] == 100
|
|
assert compute_weighted_overall(capped) < compute_weighted_overall(scores)
|
|
|
|
|
|
def make_profile(
|
|
content: str = "Python API를 개선했다.",
|
|
*,
|
|
keywords: list[str] | None = None,
|
|
) -> CandidateProfile:
|
|
return CandidateProfile(
|
|
candidate_id="candidate-1",
|
|
name="김하네스",
|
|
contact=ContactInfo(email="harness@example.com"),
|
|
facts=[
|
|
EvidenceItem(
|
|
evidence_id="ev-1",
|
|
category=EvidenceCategory.PROJECT,
|
|
content=content,
|
|
source=EvidenceSource.PORTFOLIO,
|
|
verification_status="document_verified",
|
|
keywords=keywords or [],
|
|
)
|
|
],
|
|
updated_at=NOW,
|
|
)
|
|
|
|
|
|
def test_requirement_coverage_is_priority_weighted_and_excludes_context() -> None:
|
|
analysis = JobAnalysis(
|
|
analysis_id="analysis-1",
|
|
posting_id="posting-1",
|
|
target_role="백엔드 엔지니어",
|
|
summary="직무 기준",
|
|
requirements=[
|
|
JobRequirement(
|
|
requirement_id="req-covered",
|
|
text="Python API 경험",
|
|
kind=RequirementKind.REQUIRED,
|
|
category=RequirementCategory.SKILL,
|
|
priority=5,
|
|
source_quote="Python API 경험",
|
|
classification_quote="필수 요건\nPython API 경험",
|
|
),
|
|
JobRequirement(
|
|
requirement_id="req-missing",
|
|
text="Kafka 운영 경험",
|
|
kind=RequirementKind.PREFERRED,
|
|
category=RequirementCategory.SKILL,
|
|
priority=3,
|
|
source_quote="Kafka 운영 경험",
|
|
classification_quote="우대 요건\nKafka 운영 경험",
|
|
),
|
|
JobRequirement(
|
|
requirement_id="req-context",
|
|
text="글로벌 서비스 조직",
|
|
kind=RequirementKind.CONTEXT,
|
|
category=RequirementCategory.OTHER,
|
|
priority=5,
|
|
source_quote="글로벌 서비스 조직",
|
|
),
|
|
],
|
|
analysed_at=NOW,
|
|
)
|
|
draft = ResumeDraft(
|
|
draft_id="draft-1",
|
|
candidate_id="candidate-1",
|
|
posting_id="posting-1",
|
|
title="백엔드 이력서",
|
|
sections=[
|
|
DraftSection(
|
|
section_id="projects",
|
|
section_type=SectionType.PROJECTS,
|
|
heading="프로젝트",
|
|
claims=[
|
|
DraftClaim(
|
|
claim_id="claim-1",
|
|
text="Python API 개선",
|
|
evidence_ids=["ev-1"],
|
|
requirement_ids=["req-covered"],
|
|
)
|
|
],
|
|
order=0,
|
|
)
|
|
],
|
|
generated_at=NOW,
|
|
)
|
|
|
|
evidence_map = EvidenceMap(
|
|
map_id="map-1",
|
|
posting_id="posting-1",
|
|
analysis_id="analysis-1",
|
|
matches=[
|
|
EvidenceMatch(
|
|
requirement_id="req-covered",
|
|
evidence_ids=["ev-1"],
|
|
match_type=EvidenceMatchType.DIRECT,
|
|
relevance_score=1,
|
|
rationale="직접 근거",
|
|
),
|
|
EvidenceMatch(
|
|
requirement_id="req-missing",
|
|
match_type=EvidenceMatchType.GAP,
|
|
relevance_score=0,
|
|
gap_reason="근거 없음",
|
|
),
|
|
EvidenceMatch(
|
|
requirement_id="req-context",
|
|
match_type=EvidenceMatchType.GAP,
|
|
relevance_score=0,
|
|
gap_reason="평가 제외 맥락",
|
|
),
|
|
],
|
|
generated_at=NOW,
|
|
)
|
|
|
|
coverage = compute_coverage(draft, analysis, evidence_map, make_profile())
|
|
|
|
assert coverage.evidence == 1.0
|
|
assert coverage.requirements == pytest.approx(5 / 8)
|
|
|
|
|
|
def test_overall_score_is_a_deterministic_weighted_sum() -> None:
|
|
scores = {
|
|
QualityCategory.EVIDENCE: 100,
|
|
QualityCategory.JOB_ALIGNMENT: 92,
|
|
QualityCategory.COMPLETENESS: 90,
|
|
QualityCategory.KOREAN_LANGUAGE: 92,
|
|
QualityCategory.READABILITY: 92,
|
|
QualityCategory.FORMATTING: 90,
|
|
QualityCategory.CONSISTENCY: 95,
|
|
QualityCategory.PRIVACY: 100,
|
|
}
|
|
|
|
assert compute_weighted_overall(scores) == 94.15
|
|
with pytest.raises(ValueError, match="missing weighted"):
|
|
compute_weighted_overall({QualityCategory.EVIDENCE: 100})
|
|
|
|
|
|
def test_evaluation_policy_fingerprint_changes_with_judge_prompt() -> None:
|
|
first = compute_evaluation_policy_fingerprint(
|
|
system_prompt="system-v1", evaluator_prompt="judge-v1"
|
|
)
|
|
second = compute_evaluation_policy_fingerprint(
|
|
system_prompt="system-v1", evaluator_prompt="judge-v2"
|
|
)
|
|
|
|
assert first != second
|
|
|
|
|
|
def test_requirement_coverage_ignores_unrelated_claim_text() -> None:
|
|
analysis = JobAnalysis(
|
|
analysis_id="analysis-1",
|
|
posting_id="posting-1",
|
|
target_role="백엔드 엔지니어",
|
|
summary="직무 기준",
|
|
requirements=[
|
|
JobRequirement(
|
|
requirement_id="req-python",
|
|
text="Python 개발 경험",
|
|
kind=RequirementKind.REQUIRED,
|
|
category=RequirementCategory.SKILL,
|
|
source_quote="Python 개발 경험",
|
|
classification_quote="필수\nPython 개발 경험",
|
|
)
|
|
],
|
|
analysed_at=NOW,
|
|
)
|
|
draft = ResumeDraft(
|
|
draft_id="draft-unrelated",
|
|
candidate_id="candidate-1",
|
|
posting_id="posting-1",
|
|
title="백엔드 이력서",
|
|
sections=[
|
|
DraftSection(
|
|
section_id="experience",
|
|
section_type=SectionType.EXPERIENCE,
|
|
heading="경험",
|
|
claims=[
|
|
DraftClaim(
|
|
claim_id="claim-unrelated",
|
|
text="고객 인터뷰를 수행했다",
|
|
evidence_ids=["ev-1"],
|
|
requirement_ids=["req-python"],
|
|
)
|
|
],
|
|
order=0,
|
|
)
|
|
],
|
|
generated_at=NOW,
|
|
)
|
|
evidence_map = EvidenceMap(
|
|
map_id="map-unrelated",
|
|
posting_id="posting-1",
|
|
analysis_id="analysis-1",
|
|
matches=[
|
|
EvidenceMatch(
|
|
requirement_id="req-python",
|
|
evidence_ids=["ev-1"],
|
|
match_type=EvidenceMatchType.DIRECT,
|
|
relevance_score=1,
|
|
rationale="잘못된 연결",
|
|
)
|
|
],
|
|
generated_at=NOW,
|
|
)
|
|
|
|
assert (
|
|
compute_coverage(
|
|
draft,
|
|
analysis,
|
|
evidence_map,
|
|
make_profile("고객 인터뷰를 수행했다."),
|
|
).requirements
|
|
== 0.0
|
|
)
|
|
|
|
|
|
def test_requirement_coverage_rechecks_actual_evidence_semantics() -> None:
|
|
requirement = JobRequirement(
|
|
requirement_id="req-support",
|
|
text="고객 상담 경험",
|
|
kind=RequirementKind.REQUIRED,
|
|
category=RequirementCategory.EXPERIENCE,
|
|
priority=5,
|
|
source_quote="고객 상담 경험",
|
|
classification_quote="필수 요건\n고객 상담 경험",
|
|
)
|
|
analysis = JobAnalysis(
|
|
analysis_id="analysis-support",
|
|
posting_id="posting-1",
|
|
target_role="고객 상담원",
|
|
summary="고객 상담 직무 기준",
|
|
requirements=[requirement],
|
|
analysed_at=NOW,
|
|
)
|
|
draft = ResumeDraft(
|
|
draft_id="draft-support",
|
|
candidate_id="candidate-1",
|
|
posting_id="posting-1",
|
|
title="고객 상담 이력서",
|
|
sections=[
|
|
DraftSection(
|
|
section_id="experience",
|
|
section_type=SectionType.EXPERIENCE,
|
|
heading="경험",
|
|
claims=[
|
|
DraftClaim(
|
|
claim_id="claim-support",
|
|
text="고객 상담 경험",
|
|
evidence_ids=["ev-1"],
|
|
requirement_ids=["req-support"],
|
|
)
|
|
],
|
|
order=0,
|
|
)
|
|
],
|
|
generated_at=NOW,
|
|
)
|
|
evidence_map = EvidenceMap(
|
|
map_id="map-support",
|
|
posting_id="posting-1",
|
|
analysis_id="analysis-support",
|
|
matches=[
|
|
EvidenceMatch(
|
|
requirement_id="req-support",
|
|
evidence_ids=["ev-1"],
|
|
match_type=EvidenceMatchType.DIRECT,
|
|
relevance_score=1,
|
|
rationale="고객 단어가 같다.",
|
|
)
|
|
],
|
|
generated_at=NOW,
|
|
)
|
|
|
|
coverage = compute_coverage(
|
|
draft,
|
|
analysis,
|
|
evidence_map,
|
|
make_profile("고객 명단을 정리했다."),
|
|
)
|
|
assert coverage.requirements == 0.0
|