from __future__ import annotations import json from collections.abc import Mapping from datetime import date, datetime, timezone from typing import Any import pytest from pydantic import BaseModel from resume_harness.backend import LLMBackend from resume_harness.models import ( CandidateProfile, ContactInfo, ContentPlan, ConstraintKind, DraftClaim, DraftSection, EvidenceCategory, EvidenceItem, EvidenceMap, EvidenceMatch, EvidenceMatchType, EvidenceSource, GenerationConfig, JobAnalysis, JobPosting, JobRequirement, PlannedSection, PostingConstraint, QualityCategory, QualityFinding, QualityReport, QualitySeverity, RequirementCategory, RequirementKind, ResumeDraft, ResumeMode, SectionType, SensitiveDataCategory, SensitiveDataConsent, ) from resume_harness.pipeline import PipelineError, PipelineStatus, ResumePipeline from resume_harness.records import ( CareerRecord, EmploymentType, RecordDate, RecordPeriod, ResumeRecords, ) UTC = timezone.utc NOW = datetime(2026, 7, 1, 12, tzinfo=UTC) class FakeBackend: """Strict FIFO fake: an unexpected stage fails the integration test.""" def __init__(self, responses: list[tuple[str, BaseModel | Mapping[str, Any]]]): self.responses = list(responses) self.calls: list[dict[str, Any]] = [] def complete_json( self, *, stage: str, system_prompt: str, task_prompt: str, user_payload: Mapping[str, Any], output_model: type[BaseModel], ) -> BaseModel | Mapping[str, Any]: assert system_prompt assert task_prompt assert self.responses, f"unexpected backend call for {stage}" expected_stage, response = self.responses.pop(0) assert stage == expected_stage assert isinstance(response, output_model) or isinstance(response, Mapping) self.calls.append( { "stage": stage, "payload": user_payload, "output_model": output_model, } ) return response def _inputs() -> tuple[CandidateProfile, JobPosting, GenerationConfig]: consent = SensitiveDataConsent( consent_id="consent-photo", category=SensitiveDataCategory.PHOTO, purpose="지정 양식 사진", granted_at=datetime(2026, 1, 1, tzinfo=UTC), expires_at=datetime(2027, 1, 1, tzinfo=UTC), ) facts = [ EvidenceItem( evidence_id="ev-api", category=EvidenceCategory.PROJECT, content="Python 결제 API 응답 시간을 40% 단축해 35ms로 개선했다.", source=EvidenceSource.PORTFOLIO, source_reference="https://private.example/internal/source", metrics={"latency_reduction": "40%", "latency": "35ms"}, keywords=["Python", "API"], ), EvidenceItem( evidence_id="ev-photo", category=EvidenceCategory.OTHER, content="지원자 증명사진 secret-photo-token", source=EvidenceSource.DOCUMENT, sensitive_category=SensitiveDataCategory.PHOTO, consent_id="consent-photo", ), EvidenceItem( evidence_id="ev-confidential", category=EvidenceCategory.PROJECT, content="secret-company-project 내부 수치", source=EvidenceSource.USER_STATEMENT, metrics={"latency": "35ms"}, keywords=["Python"], confidential=True, ), ] profile = CandidateProfile( candidate_id="candidate-1", name="김하네스", name_en="Harness Kim", contact=ContactInfo( email="harness@example.com", phone="010-1234-5678", city="서울", links=["https://portfolio.example/harness"], ), facts=facts, consents=[consent], updated_at=NOW, ) posting = JobPosting( posting_id="posting-1", company_name="합성테크", title="백엔드 엔지니어", raw_text=( "필수 요건\nPython 기반 API 개발 경험을 갖춘 " "백엔드 엔지니어를 채용합니다." ), collected_at=NOW, ) config = GenerationConfig(as_of_date=date(2026, 7, 1)) return profile, posting, config def _analysis() -> JobAnalysis: return JobAnalysis( analysis_id="analysis-1", posting_id="posting-1", target_role="백엔드 엔지니어", summary="Python API 개발 경험을 중시한다.", requirements=[ JobRequirement( requirement_id="req-python", text="Python 기반 API 개발 경험", kind=RequirementKind.REQUIRED, category=RequirementCategory.SKILL, priority=5, source_quote="Python 기반 API 개발 경험", classification_quote="필수 요건\nPython 기반 API 개발 경험", keywords=["Python", "API"], ) ], keywords=["Python", "API"], analysed_at=NOW, ) def _evidence_map() -> EvidenceMap: return EvidenceMap( map_id="map-1", posting_id="posting-1", analysis_id="analysis-1", matches=[ EvidenceMatch( requirement_id="req-python", evidence_ids=["ev-api"], match_type=EvidenceMatchType.DIRECT, relevance_score=0.95, rationale="Python API 개선 경험이 직접 연결된다.", ) ], generated_at=NOW, ) def _plan( mode: ResumeMode = ResumeMode.PRIVATE_MODERN, ) -> ContentPlan: return ContentPlan( plan_id="plan-1", candidate_id="candidate-1", posting_id="posting-1", mode=mode, sections=[ PlannedSection( section_id="section-projects", section_type=SectionType.PROJECTS, heading="주요 프로젝트", evidence_ids=["ev-api"], requirement_ids=["req-python"], bullet_budget=2, order=0, ) ], created_at=NOW, ) def _draft( text: str = "Python 결제 API 응답 시간을 40% 단축", *, mode: ResumeMode = ResumeMode.PRIVATE_MODERN, ) -> ResumeDraft: return ResumeDraft( draft_id="draft-1", candidate_id="candidate-1", posting_id="posting-1", title="백엔드 엔지니어 이력서", mode=mode, sections=[ DraftSection( section_id="section-projects", section_type=SectionType.PROJECTS, heading="주요 프로젝트", claims=[ DraftClaim( claim_id="claim-api", text=text, evidence_ids=["ev-api"], requirement_ids=["req-python"], order=0, ) ], order=0, ) ], generated_at=NOW, ) def _good_report(draft: ResumeDraft | None = None) -> QualityReport: bound_draft = draft or _draft() return QualityReport( report_id="report-good", draft_id="draft-1", draft_fingerprint=bound_draft.fingerprint(), overall_score=94, evidence_coverage=1.0, requirement_coverage=1.0, category_scores={ QualityCategory.EVIDENCE: 100, QualityCategory.JOB_ALIGNMENT: 92, QualityCategory.COMPLETENESS: 92, QualityCategory.KOREAN_LANGUAGE: 91, QualityCategory.READABILITY: 92, QualityCategory.FORMATTING: 95, QualityCategory.CONSISTENCY: 95, QualityCategory.PRIVACY: 100, }, findings=[], evaluated_at=NOW, ) def _bad_report( report_id: str, draft: ResumeDraft | None = None ) -> QualityReport: bound_draft = draft or _draft() return QualityReport( report_id=report_id, draft_id="draft-1", draft_fingerprint=bound_draft.fingerprint(), overall_score=84, evidence_coverage=1.0, requirement_coverage=1.0, category_scores={ QualityCategory.EVIDENCE: 100, QualityCategory.JOB_ALIGNMENT: 76, QualityCategory.COMPLETENESS: 82, QualityCategory.KOREAN_LANGUAGE: 78, QualityCategory.READABILITY: 80, QualityCategory.FORMATTING: 90, QualityCategory.CONSISTENCY: 88, QualityCategory.PRIVACY: 100, }, findings=[ QualityFinding( finding_id=f"finding-{report_id}", code="STYLE.ABSTRACT_ACTION", severity=QualitySeverity.WARNING, category=QualityCategory.KOREAN_LANGUAGE, message="행동과 결과의 연결을 더 분명히 해야 한다.", claim_id="claim-api", evidence_ids=["ev-api"], suggestion="근거 범위에서 행동을 명확히 한다.", ) ], evaluated_at=NOW, ) def _base_responses( final_report: QualityReport | Mapping[str, Any], ) -> list[tuple[str, BaseModel | Mapping[str, Any]]]: return [ ("analyze-job", _analysis()), ("map-evidence", _evidence_map()), ("plan-content", _plan()), ("draft-resume", _draft()), ("evaluate-resume", final_report), ] def test_success_orders_stages_and_never_sends_candidate_pii() -> None: profile, posting, config = _inputs() # A provider adapter may return a mapping produced from a model. Computed # read-only fields in that mapping are tolerated and the writable fields are # still validated strictly. backend = FakeBackend(_base_responses(_good_report().model_dump(mode="json"))) assert isinstance(backend, LLMBackend) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.PASSED assert result.repair_attempts == 0 assert [call["stage"] for call in backend.calls] == [ "analyze-job", "map-evidence", "plan-content", "draft-resume", "evaluate-resume", ] assert backend.responses == [] transmitted = json.dumps( [call["payload"] for call in backend.calls], ensure_ascii=False ) for forbidden in ( "김하네스", "Harness Kim", "harness@example.com", "010-1234-5678", "01012345678", "https://portfolio.example/harness", "서울", "secret-photo-token", "secret-company-project", ): assert forbidden not in transmitted # Shared technical and numeric values from an excluded fact are not privacy # tokens and must remain usable in an allowed fact. map_payload = backend.calls[1]["payload"] assert map_payload["candidate_facts"][0]["keywords"] == ["Python", "API"] assert map_payload["candidate_facts"][0]["metrics"]["latency"] == "35ms" assert "source_reference" not in map_payload["candidate_facts"][0] assert "consent_id" not in map_payload["candidate_facts"][0] assert "confidential" not in map_payload["candidate_facts"][0] def test_public_blind_school_fact_is_removed_before_llm_boundary() -> None: profile, posting, _ = _inputs() profile = profile.model_copy( update={ "facts": [ *profile.facts, EvidenceItem( evidence_id="ev-school", category=EvidenceCategory.EDUCATION, content="서울대에서 컴퓨터공학을 전공했다.", source=EvidenceSource.DOCUMENT, keywords=["서울대", "컴퓨터공학"], ), EvidenceItem( evidence_id="ev-school-snu", category=EvidenceCategory.EDUCATION, content="SNU 컴퓨터공학 과정을 졸업했다.", source=EvidenceSource.DOCUMENT, keywords=["SNU", "컴퓨터공학"], ), ] } ) config = GenerationConfig( resume_mode=ResumeMode.PUBLIC_BLIND, as_of_date=date(2026, 7, 1), ) blind_draft = _draft(mode=ResumeMode.PUBLIC_BLIND) backend = FakeBackend( [ ("analyze-job", _analysis()), ("map-evidence", _evidence_map()), ("plan-content", _plan(ResumeMode.PUBLIC_BLIND)), ("draft-resume", blind_draft), ("evaluate-resume", _good_report(blind_draft)), ] ) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.PASSED transmitted = json.dumps( [call["payload"] for call in backend.calls], ensure_ascii=False ) assert "ev-school" not in transmitted assert "서울대" not in transmitted assert "ev-school-snu" not in transmitted assert "SNU" not in transmitted def test_public_blind_origin_fact_is_removed_before_llm_boundary() -> None: profile, posting, _ = _inputs() profile = profile.model_copy( update={ "facts": [ *profile.facts, EvidenceItem( evidence_id="ev-origin", category=EvidenceCategory.PROJECT, content="고향은 대전이며 Python API를 개발했다.", source=EvidenceSource.USER_STATEMENT, keywords=["Python", "API"], ), EvidenceItem( evidence_id="ev-hometown-compact", category=EvidenceCategory.PROJECT, content="고향 대전, Python API를 개발했다.", source=EvidenceSource.USER_STATEMENT, keywords=["Python", "API"], ), EvidenceItem( evidence_id="ev-grown", category=EvidenceCategory.PROJECT, content="대전에서 자랐고 Python API를 개발했다.", source=EvidenceSource.USER_STATEMENT, keywords=["Python", "API"], ), EvidenceItem( evidence_id="ev-region-work", category=EvidenceCategory.PROJECT, content="대전 지역 고객을 위한 Python API를 개발했다.", source=EvidenceSource.PORTFOLIO, keywords=["Python", "API", "대전 지역"], ), ] } ) config = GenerationConfig( resume_mode=ResumeMode.PUBLIC_BLIND, as_of_date=date(2026, 7, 1), ) blind_draft = _draft(mode=ResumeMode.PUBLIC_BLIND) backend = FakeBackend( [ ("analyze-job", _analysis()), ("map-evidence", _evidence_map()), ("plan-content", _plan(ResumeMode.PUBLIC_BLIND)), ("draft-resume", blind_draft), ("evaluate-resume", _good_report(blind_draft)), ] ) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.PASSED transmitted = json.dumps( [call["payload"] for call in backend.calls], ensure_ascii=False ) assert "ev-origin" not in transmitted assert "고향은 대전" not in transmitted assert "ev-hometown-compact" not in transmitted assert "고향 대전" not in transmitted assert "ev-grown" not in transmitted assert "대전에서 자랐고" not in transmitted assert "ev-region-work" in transmitted assert "대전 지역 고객" in transmitted def test_korean_name_with_postposition_is_redacted_before_backend() -> None: profile, posting, config = _inputs() posting = posting.model_copy( update={ "raw_text": ( "김하네스는 외부 입력입니다.\n필수 요건\n" "Python 기반 API 개발 경험을 " "갖춘 지원자를 찾습니다." ) } ) backend = FakeBackend(_base_responses(_good_report())) ResumePipeline(backend).run(profile, posting, config) analyze_payload = json.dumps(backend.calls[0]["payload"], ensure_ascii=False) assert "김하네스" not in analyze_payload assert "[REDACTED]는" in analyze_payload def test_short_korean_name_does_not_redact_an_ordinary_verb() -> None: profile, posting, config = _inputs() profile = profile.model_copy(update={"name": "이수"}) posting = posting.model_copy( update={ "raw_text": ( "교육 과정을 이수했다면 우대합니다.\n" "필수 요건\n" "Python 기반 API 개발 경험을 확인합니다." ) } ) backend = FakeBackend(_base_responses(_good_report())) ResumePipeline(backend).run(profile, posting, config) analyze_payload = json.dumps(backend.calls[0]["payload"], ensure_ascii=False) assert "이수했다" in analyze_payload assert "[REDACTED]했다" not in analyze_payload def test_no_generation_safe_evidence_returns_early_needs_user_input() -> None: profile, posting, config = _inputs() profile = profile.model_copy( update={ "facts": [ fact.model_copy(update={"confidential": True}) for fact in profile.facts ] } ) backend = FakeBackend([("analyze-job", _analysis())]) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.NEEDS_USER_INPUT assert result.gate_failures == ["no_generation_safe_evidence"] assert result.evidence_map is None assert result.draft is None assert [call["stage"] for call in backend.calls] == ["analyze-job"] def test_all_gap_evidence_map_returns_before_impossible_draft() -> None: profile, posting, config = _inputs() gap_map = EvidenceMap( map_id="map-gap", posting_id="posting-1", analysis_id="analysis-1", matches=[ EvidenceMatch( requirement_id="req-python", match_type=EvidenceMatchType.GAP, relevance_score=0, gap_reason="직접 근거가 없다.", ) ], generated_at=NOW, ) backend = FakeBackend( [("analyze-job", _analysis()), ("map-evidence", gap_map)] ) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.NEEDS_USER_INPUT assert result.gate_failures == ["all_requirements_gap"] assert result.evidence_map == gap_map assert result.content_plan is None assert result.questions assert [call["stage"] for call in backend.calls] == [ "analyze-job", "map-evidence", ] def test_posting_specific_company_name_fact_is_withheld_before_mapping() -> None: profile, posting, config = _inputs() profile = profile.model_copy( update={ "facts": [ fact.model_copy( update={ "content": "가상페이에서 Python 기반 API 개발 경험을 쌓았다." } ) if fact.evidence_id == "ev-api" else fact for fact in profile.facts ] } ) posting = posting.model_copy( update={"raw_text": posting.raw_text + "\n회사명 기재 금지"} ) analysis = _analysis().model_copy( update={ "constraints": [ PostingConstraint( constraint_id="blind-company", kind=ConstraintKind.BLIND_FIELD, description="회사명을 본문에 기재하지 않는다.", source_quote="회사명 기재 금지", fields=["회사명"], ) ] } ) backend = FakeBackend([("analyze-job", analysis)]) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.NEEDS_USER_INPUT assert result.gate_failures == ["no_generation_safe_evidence"] assert [call["stage"] for call in backend.calls] == ["analyze-job"] def test_unknown_company_fact_is_withheld_even_with_structured_career() -> None: profile, posting, config = _inputs() facts = [ fact.model_copy( update={"content": "가상페이에서 Python API를 개발했다."} ) if fact.evidence_id == "ev-api" else fact for fact in profile.facts ] facts.append( EvidenceItem( evidence_id="ev-career", category=EvidenceCategory.CAREER, content="2024년부터 기록회사 백엔드 엔지니어로 근무했다.", source=EvidenceSource.EMPLOYMENT_RECORD, date_range={"start": {"year": 2024}, "ongoing": True}, ) ) profile = profile.model_copy( update={ "facts": facts, "records": ResumeRecords( careers=[ CareerRecord( record_id="career-1", organization="기록회사", role="백엔드 엔지니어", period=RecordPeriod( start=RecordDate(year=2024), ongoing=True ), employment_type=EmploymentType.FULL_TIME, evidence_ids=["ev-career"], ) ] ), } ) posting = posting.model_copy( update={"raw_text": posting.raw_text + "\n회사명 기재 금지"} ) analysis = _analysis().model_copy( update={ "constraints": [ PostingConstraint( constraint_id="blind-company-with-records", kind=ConstraintKind.BLIND_FIELD, description="회사명을 본문에 기재하지 않는다.", source_quote="회사명 기재 금지", fields=["회사명"], ) ] } ) backend = FakeBackend([("analyze-job", analysis)]) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.NEEDS_USER_INPUT assert result.gate_failures == ["no_generation_safe_evidence"] assert [call["stage"] for call in backend.calls] == ["analyze-job"] def test_claim_finding_triggers_targeted_repair_and_re_evaluation() -> None: profile, posting, config = _inputs() repaired = _draft("Python 결제 API 응답 시간을 40% 단축") backend = FakeBackend( [ *_base_responses(_bad_report("report-before")), ("repair-resume", repaired), ("evaluate-resume", _good_report(repaired)), ] ) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.PASSED assert result.repair_attempts == 1 assert result.draft.sections[0].claims[0].text == repaired.sections[0].claims[0].text assert [call["stage"] for call in backend.calls] == [ "analyze-job", "map-evidence", "plan-content", "draft-resume", "evaluate-resume", "repair-resume", "evaluate-resume", ] repair_payload = backend.calls[5]["payload"] assert [item["claim_id"] for item in repair_payload["approved_findings"]] == [ "claim-api" ] def test_two_failed_repairs_return_needs_user_input_with_bounded_questions() -> None: profile, posting, config = _inputs() repaired_once = _draft("Python 결제 API 응답 시간을 40% 단축함") repaired_twice = _draft("결제 API 응답 시간을 Python으로 40% 단축") backend = FakeBackend( [ *_base_responses(_bad_report("report-0")), ("repair-resume", repaired_once), ("evaluate-resume", _bad_report("report-1", repaired_once)), ("repair-resume", repaired_twice), ("evaluate-resume", _bad_report("report-2", repaired_twice)), ] ) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.NEEDS_USER_INPUT assert result.repair_attempts == 2 assert 1 <= len(result.questions) <= 3 assert result.gate_failures assert [call["stage"] for call in backend.calls].count("repair-resume") == 2 assert [call["stage"] for call in backend.calls].count("evaluate-resume") == 3 assert backend.responses == [] def test_judge_cannot_inflate_deterministic_requirement_coverage() -> None: profile, posting, config = _inputs() uncovered = _draft() uncovered.sections[0].claims[0].requirement_ids = [] backend = FakeBackend( [ ("analyze-job", _analysis()), ("map-evidence", _evidence_map()), ("plan-content", _plan()), ("draft-resume", uncovered), ("evaluate-resume", _good_report(uncovered)), ] ) result = ResumePipeline(backend, max_repair_attempts=0).run( profile, posting, config ) assert result.status is PipelineStatus.NEEDS_USER_INPUT assert result.quality_report is not None assert result.quality_report.requirement_coverage == 0 assert any( failure.startswith("requirement_coverage:0<") for failure in result.gate_failures ) def test_high_judge_score_cannot_release_unrelated_korean_claim() -> None: profile, posting, config = _inputs() hallucinated = _draft( "고객 만족도를 혁신적으로 높이고 조직 문화를 획기적으로 개선했다" ) backend = FakeBackend( [ ("analyze-job", _analysis()), ("map-evidence", _evidence_map()), ("plan-content", _plan()), ("draft-resume", hallucinated), ("evaluate-resume", _good_report(hallucinated)), ] ) result = ResumePipeline(backend, max_repair_attempts=0).run( profile, posting, config ) assert result.status is PipelineStatus.NEEDS_USER_INPUT assert "GROUNDING.LOW_LEXICAL_SUPPORT" in { finding.code for finding in result.deterministic_findings } def test_release_pipeline_rejects_non_strict_evidence_mode() -> None: profile, posting, config = _inputs() config = config.model_copy(update={"strict_evidence": False}) backend = FakeBackend([]) with pytest.raises(PipelineError, match="strict_evidence=true"): ResumePipeline(backend).run(profile, posting, config) assert backend.calls == [] def test_job_analysis_constraints_participate_in_deterministic_gate() -> None: profile, posting, config = _inputs() posting = posting.model_copy( update={"raw_text": posting.raw_text + " 학교명 기재 금지"} ) analysis = _analysis().model_copy( update={ "constraints": [ PostingConstraint( constraint_id="constraint-school", kind=ConstraintKind.BLIND_FIELD, description="학교명을 본문에 기재하지 않는다.", source_quote="학교명 기재 금지", fields=["학교명"], ) ] } ) constrained_draft = _draft( "학교명: 합성대학교에서 Python 결제 API 응답 시간을 40% 단축" ) backend = FakeBackend( [ ("analyze-job", analysis), ("map-evidence", _evidence_map()), ("plan-content", _plan()), ("draft-resume", constrained_draft), ("evaluate-resume", _good_report(constrained_draft)), ] ) result = ResumePipeline(backend, max_repair_attempts=0).run( profile, posting, config ) assert result.status is PipelineStatus.NEEDS_USER_INPUT assert "PRIVACY.POSTING_FIELD_LEAK" in { finding.code for finding in result.deterministic_findings } def test_consented_employer_form_sensitive_fact_stays_renderer_only() -> None: profile, posting, _ = _inputs() config = GenerationConfig( resume_mode=ResumeMode.EMPLOYER_FORM, as_of_date=date(2026, 7, 1), include_photo=True, allowed_sensitive_categories={SensitiveDataCategory.PHOTO}, employer_required_sensitive_categories={SensitiveDataCategory.PHOTO}, ) employer_draft = _draft(mode=ResumeMode.EMPLOYER_FORM) backend = FakeBackend( [ ("analyze-job", _analysis()), ("map-evidence", _evidence_map()), ("plan-content", _plan(ResumeMode.EMPLOYER_FORM)), ("draft-resume", employer_draft), ("evaluate-resume", _good_report(employer_draft)), ] ) result = ResumePipeline(backend).run(profile, posting, config) assert result.status is PipelineStatus.PASSED transmitted = json.dumps( [call["payload"] for call in backend.calls], ensure_ascii=False ) assert "ev-photo" not in transmitted assert "secret-photo-token" not in transmitted