#!/usr/bin/env python3 import hashlib import json import shutil import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "render_world_class_evidence_ledger.py" TMP = ROOT / "tests" / "tmp_world_class_evidence_ledger" def sha256_file(path: Path) -> str: digest = hashlib.sha256() with path.open("rb") as handle: for chunk in iter(lambda: handle.read(1024 * 1024), b""): digest.update(chunk) return digest.hexdigest() def provider_submission(artifact_root: Path = ROOT, artifact_path: str = "reports/output_execution_runs.json") -> dict: return { "schema_version": "1.0", "evidence_key": "provider-holdout", "template_only": False, "category": "external", "source_type": "provider-output-eval", "submitted_by": "Yao provider operator", "submitted_at": "2026-06-13", "summary": "Aggregate provider-backed holdout evidence for ledger review.", "artifact_refs": [ { "path": artifact_path, "kind": "aggregate-report", "contains_raw_content": False, "sha256": sha256_file(artifact_root / artifact_path), } ], "provenance": { "provider": "deepseek", "model": "deepseek-v4-flash", "credential_material_committed": False, }, "privacy": { "raw_user_content_included": False, "raw_provider_prompt_included": False, "credentials_included": False, "secrets_included": False, }, "anti_overclaim": { "planned_work_counts_as_evidence": False, "metadata_fallback_counts_as_native_enforcement": False, "pending_review_counts_as_human_decision": False, "local_command_runner_counts_as_provider_model": False, }, "attestation": { "real_external_or_human_evidence": True, "reviewer_or_operator_identity_present": True, "artifact_refs_reviewed": True, "privacy_contract_satisfied": True, "ledger_reviewer_approved": True, "ledger_reviewer": "Yao ledger reviewer", "ledger_reviewed_at": "2026-06-13", }, } def accepted_provider_execution_report() -> dict: return { "schema_version": "1.0", "ok": True, "summary": { "variant_run_count": 1, "model_executed_count": 1, "timing_observed_count": 1, "token_observed_count": 1, "failure_count": 0, }, "runs": [ { "case_id": "provider-holdout", "variant": "with_skill", "status": "pass", "execution_mode": "model", "model_executed": True, "command_executed": True, "duration_ms": 1234.5, "provider": "deepseek", "model": "deepseek-v4-flash", "usage": { "input_tokens": 20, "output_tokens": 30, "total_tokens": 50, "estimated": False, }, "output_sha256": "a" * 64, "failure": "", } ], } def main() -> None: shutil.rmtree(TMP, ignore_errors=True) TMP.mkdir(parents=True, exist_ok=True) empty_submissions = TMP / "empty-submissions" empty_submissions.mkdir() output_json = TMP / "world_class_evidence_ledger.json" output_md = TMP / "world_class_evidence_ledger.md" proc = subprocess.run( [ sys.executable, str(SCRIPT), str(ROOT), "--output-json", str(output_json), "--output-md", str(output_md), "--submissions-dir", str(empty_submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) payload = json.loads(proc.stdout) assert payload["schema_version"] == "1.0", payload assert payload["ok"] is True, payload summary = payload["summary"] assert payload["report_contract"]["contract"] == "world-class-evidence-ledger", payload assert payload["report_contract"]["top_level_mirrors_summary"] is True, payload for field in payload["report_contract"]["summary_fields"]: assert payload[field] == summary[field], (field, payload[field], summary[field]) assert summary["ledger_entry_count"] == 4, summary assert summary["accepted_count"] == 0, summary assert summary["pending_count"] == 4, summary assert summary["human_pending_count"] == 1, summary assert summary["external_pending_count"] == 3, summary assert summary["submitted_entry_count"] == 0, summary assert summary["missing_submission_count"] == 4, summary assert summary["invalid_submission_count"] == 0, summary assert summary["source_check_count"] >= 13, summary assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary assert summary["source_blocked_count"] >= 6, summary assert summary["submitted_but_pending_count"] == 0, summary assert summary["overclaim_guard_active"] is True, summary assert summary["ready_to_claim_world_class"] is False, summary assert payload["ready_to_claim_world_class"] is False, payload assert payload["pending_count"] == 4, payload assert payload["accepted_count"] == 0, payload assert payload["source_blocked_count"] == summary["source_blocked_count"], payload assert payload["artifacts"]["intake"] == "reports/world_class_evidence_intake.md", payload entries = {entry["key"]: entry for entry in payload["entries"]} assert set(entries) == { "provider-holdout", "human-adjudication", "native-permission-enforcement", "native-client-telemetry", }, entries assert entries["provider-holdout"]["source_accepted"] is True, entries["provider-holdout"] assert entries["provider-holdout"]["observed_state"]["model_executed_count"] > 0, entries["provider-holdout"] assert any("--provider-runner openai" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] assert any("--provider-runner deepseek" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] assert not any("" in step or "OPENAI_API_KEY=" in step for step in entries["provider-holdout"]["runbook"]), entries["provider-holdout"] provider_source = {item["field"]: item for item in entries["provider-holdout"]["source_checklist"]} assert provider_source["model_executed_count"]["status"] == "pass", provider_source assert provider_source["timing_observed_count"]["status"] == "pass", provider_source assert provider_source["token_observed_count"]["status"] == "pass", provider_source assert entries["provider-holdout"]["submission_state"]["status"] == "missing", entries["provider-holdout"] assert entries["provider-holdout"]["submission_state"]["ledger_counts_as_completion"] is False, entries["provider-holdout"] assert entries["human-adjudication"]["observed_state"]["pending_count"] == 5, entries["human-adjudication"] assert entries["human-adjudication"]["observed_state"]["reviewer_metadata_present"] is False, entries["human-adjudication"] assert entries["human-adjudication"]["observed_state"]["reason_required"] is True, entries["human-adjudication"] assert entries["human-adjudication"]["observed_state"]["raw_content_allowed"] is False, entries["human-adjudication"] assert entries["human-adjudication"]["observed_state"]["raw_content_path_count"] == 0, entries["human-adjudication"] assert entries["human-adjudication"]["observed_state"]["ready_for_human_evidence"] is False, entries["human-adjudication"] human_source = {item["field"]: item for item in entries["human-adjudication"]["source_checklist"]} assert human_source["reviewer_metadata_present"]["status"] == "blocked", human_source assert human_source["reason_required"]["status"] == "pass", human_source assert human_source["raw_content_allowed"]["status"] == "pass", human_source assert human_source["ready_for_human_evidence"]["status"] == "blocked", human_source assert "choice and reason" in human_source["pending_count"]["next_action"], human_source assert "metadata and rationale" in human_source["ready_for_human_evidence"]["next_action"], human_source assert entries["native-permission-enforcement"]["observed_state"]["native_enforcement_count"] == 0, entries["native-permission-enforcement"] assert entries["native-permission-enforcement"]["observed_state"]["installer_enforcement_pass_count"] >= 0, entries["native-permission-enforcement"] assert any("summary.failure_count == 0" in check for check in entries["native-permission-enforcement"]["success_checks"]), entries["native-permission-enforcement"] assert entries["native-client-telemetry"]["observed_state"]["external_source_events"] == 0, entries["native-client-telemetry"] for entry in entries.values(): assert entry["status"] == "pending", entry assert entry["success_checks"], entry assert entry["privacy_contract"], entry assert entry["anti_overclaim"]["planned_work_counts_as_evidence"] is False, entry assert entry["anti_overclaim"]["pending_review_counts_as_human_decision"] is False, entry assert "submission_state" in entry, entry markdown = output_md.read_text(encoding="utf-8") assert "World-Class Evidence Ledger" in markdown, markdown assert "overclaim guard active: `true`" in markdown, markdown assert "submitted entries: `0`" in markdown, markdown assert "source checks:" in markdown, markdown assert "Source Runbook" in markdown, markdown assert "--provider-runner openai" in markdown, markdown assert "--provider-runner deepseek" in markdown, markdown assert "" not in markdown, markdown assert "OPENAI_API_KEY=" not in markdown, markdown assert "Source Evidence Checks" in markdown, markdown assert "| Provider model run | `10` | `>0` | `pass` |" in markdown, markdown assert "`provider-holdout`" in markdown, markdown submissions = TMP / "submissions" submissions.mkdir() (submissions / "provider-holdout.json").write_text( json.dumps(provider_submission(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) submitted_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(ROOT), "--output-json", str(TMP / "submitted_ledger.json"), "--output-md", str(TMP / "submitted_ledger.md"), "--submissions-dir", str(submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) submitted_payload = json.loads(submitted_proc.stdout) submitted_summary = submitted_payload["summary"] assert submitted_summary["submitted_entry_count"] == 1, submitted_summary assert submitted_summary["submitted_but_pending_count"] == 0, submitted_summary assert submitted_summary["invalid_submission_count"] == 0, submitted_summary assert submitted_summary["accepted_count"] == 1, submitted_summary assert submitted_summary["pending_count"] == 3, submitted_summary assert submitted_summary["source_blocked_count"] >= 6, submitted_summary submitted_provider = {entry["key"]: entry for entry in submitted_payload["entries"]}["provider-holdout"] assert submitted_provider["status"] == "accepted", submitted_provider assert submitted_provider["submission_state"]["status"] == "submitted", submitted_provider assert submitted_provider["submission_state"]["artifact_sha256_verified_count"] == 1, submitted_provider assert submitted_provider["submission_state"]["attested_real_evidence"] is True, submitted_provider assert submitted_provider["submission_state"]["ledger_counts_as_completion"] is False, submitted_provider assert submitted_provider["submission_state"]["errors"] == [], submitted_provider accepted_source_skill = TMP / "accepted_source_skill" (accepted_source_skill / "reports").mkdir(parents=True) (accepted_source_skill / "reports" / "output_execution_runs.json").write_text( json.dumps(accepted_provider_execution_report(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) accepted_source_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(accepted_source_skill), "--output-json", str(TMP / "accepted_source_without_submission.json"), "--output-md", str(TMP / "accepted_source_without_submission.md"), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) accepted_source_payload = json.loads(accepted_source_proc.stdout) accepted_source_summary = accepted_source_payload["summary"] assert accepted_source_summary["source_accepted_count"] == 1, accepted_source_summary assert accepted_source_summary["accepted_count"] == 0, accepted_source_summary assert accepted_source_summary["source_accepted_without_valid_submission_count"] == 1, accepted_source_summary assert accepted_source_summary["source_check_count"] >= 13, accepted_source_summary accepted_source_provider = { entry["key"]: entry for entry in accepted_source_payload["entries"] }["provider-holdout"] assert accepted_source_provider["source_accepted"] is True, accepted_source_provider assert accepted_source_provider["status"] == "pending", accepted_source_provider assert accepted_source_provider["submission_state"]["status"] == "missing", accepted_source_provider assert all(item["status"] == "pass" for item in accepted_source_provider["source_checklist"]), accepted_source_provider (accepted_source_skill / "reports" / "context_budget.json").write_text( json.dumps({"summary": {"unrelated": True}}, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) unrelated_accepted_submissions = TMP / "unrelated_accepted_submissions" unrelated_accepted_submissions.mkdir() (unrelated_accepted_submissions / "provider-holdout.json").write_text( json.dumps( provider_submission(accepted_source_skill, artifact_path="reports/context_budget.json"), ensure_ascii=False, indent=2, ) + "\n", encoding="utf-8", ) unrelated_accepted_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(accepted_source_skill), "--output-json", str(TMP / "unrelated_accepted_provider_ledger.json"), "--output-md", str(TMP / "unrelated_accepted_provider_ledger.md"), "--submissions-dir", str(unrelated_accepted_submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) unrelated_accepted_payload = json.loads(unrelated_accepted_proc.stdout) unrelated_accepted_summary = unrelated_accepted_payload["summary"] assert unrelated_accepted_summary["source_accepted_count"] == 1, unrelated_accepted_summary assert unrelated_accepted_summary["accepted_count"] == 0, unrelated_accepted_summary assert unrelated_accepted_summary["invalid_submission_count"] == 1, unrelated_accepted_summary unrelated_accepted_provider = { entry["key"]: entry for entry in unrelated_accepted_payload["entries"] }["provider-holdout"] assert unrelated_accepted_provider["source_accepted"] is True, unrelated_accepted_provider assert unrelated_accepted_provider["status"] == "pending", unrelated_accepted_provider assert unrelated_accepted_provider["submission_state"]["status"] == "invalid-contract", unrelated_accepted_provider assert any( "required evidence artifact reports/output_execution_runs.json" in error for error in unrelated_accepted_provider["submission_state"]["errors"] ), unrelated_accepted_provider placeholder_accepted_submissions = TMP / "placeholder_accepted_submissions" placeholder_accepted_submissions.mkdir() placeholder_submission = provider_submission(accepted_source_skill) placeholder_submission["submitted_by"] = "operator with provider credentials" placeholder_submission["submitted_at"] = "YYYY-MM-DD" (placeholder_accepted_submissions / "provider-holdout.json").write_text( json.dumps(placeholder_submission, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) placeholder_accepted_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(accepted_source_skill), "--output-json", str(TMP / "placeholder_accepted_provider_ledger.json"), "--output-md", str(TMP / "placeholder_accepted_provider_ledger.md"), "--submissions-dir", str(placeholder_accepted_submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) placeholder_accepted_payload = json.loads(placeholder_accepted_proc.stdout) placeholder_accepted_summary = placeholder_accepted_payload["summary"] assert placeholder_accepted_summary["source_accepted_count"] == 1, placeholder_accepted_summary assert placeholder_accepted_summary["accepted_count"] == 0, placeholder_accepted_summary assert placeholder_accepted_summary["invalid_submission_count"] == 1, placeholder_accepted_summary placeholder_accepted_provider = { entry["key"]: entry for entry in placeholder_accepted_payload["entries"] }["provider-holdout"] assert placeholder_accepted_provider["source_accepted"] is True, placeholder_accepted_provider assert placeholder_accepted_provider["status"] == "pending", placeholder_accepted_provider assert placeholder_accepted_provider["submission_state"]["status"] == "invalid-contract", placeholder_accepted_provider assert any( "submitted_by must not use template placeholder text" in error for error in placeholder_accepted_provider["submission_state"]["errors"] ), placeholder_accepted_provider unapproved_submissions = TMP / "unapproved_submissions" unapproved_submissions.mkdir() unapproved_submission = provider_submission(accepted_source_skill) unapproved_submission["attestation"]["ledger_reviewer_approved"] = False (unapproved_submissions / "provider-holdout.json").write_text( json.dumps(unapproved_submission, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) unapproved_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(accepted_source_skill), "--output-json", str(TMP / "unapproved_provider_ledger.json"), "--output-md", str(TMP / "unapproved_provider_ledger.md"), "--submissions-dir", str(unapproved_submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) unapproved_payload = json.loads(unapproved_proc.stdout) unapproved_summary = unapproved_payload["summary"] assert unapproved_summary["source_accepted_count"] == 1, unapproved_summary assert unapproved_summary["submitted_entry_count"] == 0, unapproved_summary assert unapproved_summary["reviewer_approved_submission_count"] == 0, unapproved_summary assert unapproved_summary["accepted_count"] == 0, unapproved_summary unapproved_provider = { entry["key"]: entry for entry in unapproved_payload["entries"] }["provider-holdout"] assert unapproved_provider["source_accepted"] is True, unapproved_provider assert unapproved_provider["status"] == "pending", unapproved_provider assert unapproved_provider["submission_state"]["status"] == "invalid-contract", unapproved_provider assert unapproved_provider["submission_state"]["ledger_reviewer_approved"] is False, unapproved_provider assert any( "attestation.ledger_reviewer_approved must be true" in error for error in unapproved_provider["submission_state"]["errors"] ), unapproved_provider missing_reviewer_submissions = TMP / "missing_reviewer_submissions" missing_reviewer_submissions.mkdir() missing_reviewer_submission = provider_submission(accepted_source_skill) missing_reviewer_submission["attestation"]["ledger_reviewer"] = "" (missing_reviewer_submissions / "provider-holdout.json").write_text( json.dumps(missing_reviewer_submission, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) missing_reviewer_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(accepted_source_skill), "--output-json", str(TMP / "missing_reviewer_provider_ledger.json"), "--output-md", str(TMP / "missing_reviewer_provider_ledger.md"), "--submissions-dir", str(missing_reviewer_submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) missing_reviewer_payload = json.loads(missing_reviewer_proc.stdout) missing_reviewer_provider = { entry["key"]: entry for entry in missing_reviewer_payload["entries"] }["provider-holdout"] assert missing_reviewer_provider["status"] == "pending", missing_reviewer_provider assert missing_reviewer_provider["submission_state"]["status"] == "invalid-contract", missing_reviewer_provider assert any( "attestation.ledger_reviewer is required" in error for error in missing_reviewer_provider["submission_state"]["errors"] ), missing_reviewer_provider self_approved_submissions = TMP / "self_approved_submissions" self_approved_submissions.mkdir() self_approved_submission = provider_submission(accepted_source_skill) self_approved_submission["attestation"]["ledger_reviewer"] = self_approved_submission["submitted_by"] (self_approved_submissions / "provider-holdout.json").write_text( json.dumps(self_approved_submission, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) self_approved_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(accepted_source_skill), "--output-json", str(TMP / "self_approved_provider_ledger.json"), "--output-md", str(TMP / "self_approved_provider_ledger.md"), "--submissions-dir", str(self_approved_submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) self_approved_payload = json.loads(self_approved_proc.stdout) self_approved_provider = { entry["key"]: entry for entry in self_approved_payload["entries"] }["provider-holdout"] assert self_approved_provider["status"] == "pending", self_approved_provider assert self_approved_provider["submission_state"]["status"] == "invalid-contract", self_approved_provider assert any( "attestation.ledger_reviewer must be different from submitted_by" in error for error in self_approved_provider["submission_state"]["errors"] ), self_approved_provider early_review_submissions = TMP / "early_review_submissions" early_review_submissions.mkdir() early_review_submission = provider_submission(accepted_source_skill) early_review_submission["submitted_at"] = "2026-06-13" early_review_submission["attestation"]["ledger_reviewed_at"] = "2026-06-12" (early_review_submissions / "provider-holdout.json").write_text( json.dumps(early_review_submission, ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) early_review_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(accepted_source_skill), "--output-json", str(TMP / "early_review_provider_ledger.json"), "--output-md", str(TMP / "early_review_provider_ledger.md"), "--submissions-dir", str(early_review_submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) early_review_payload = json.loads(early_review_proc.stdout) early_review_provider = { entry["key"]: entry for entry in early_review_payload["entries"] }["provider-holdout"] assert early_review_provider["source_accepted"] is True, early_review_provider assert early_review_provider["status"] == "pending", early_review_provider assert early_review_provider["submission_state"]["status"] == "invalid-contract", early_review_provider assert any( "attestation.ledger_reviewed_at must be at or after submitted_at" in error for error in early_review_provider["submission_state"]["errors"] ), early_review_provider accepted_submissions = TMP / "accepted_submissions" accepted_submissions.mkdir() (accepted_submissions / "provider-holdout.json").write_text( json.dumps(provider_submission(accepted_source_skill), ensure_ascii=False, indent=2) + "\n", encoding="utf-8", ) accepted_proc = subprocess.run( [ sys.executable, str(SCRIPT), str(accepted_source_skill), "--output-json", str(TMP / "accepted_provider_ledger.json"), "--output-md", str(TMP / "accepted_provider_ledger.md"), "--submissions-dir", str(accepted_submissions), "--generated-at", "2026-06-13", ], cwd=ROOT, capture_output=True, text=True, check=True, ) accepted_payload = json.loads(accepted_proc.stdout) accepted_summary = accepted_payload["summary"] assert accepted_summary["source_accepted_count"] == 1, accepted_summary assert accepted_summary["accepted_count"] == 1, accepted_summary assert accepted_summary["reviewer_approved_submission_count"] == 1, accepted_summary assert accepted_summary["pending_count"] == 3, accepted_summary assert accepted_summary["source_accepted_without_valid_submission_count"] == 0, accepted_summary accepted_provider = {entry["key"]: entry for entry in accepted_payload["entries"]}["provider-holdout"] assert accepted_provider["status"] == "accepted", accepted_provider assert accepted_provider["submission_state"]["status"] == "submitted", accepted_provider print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()