{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:ULYM7QQU64RXMJUFQFXFSLC2RM","short_pith_number":"pith:ULYM7QQU","schema_version":"1.0","canonical_sha256":"a2f0cfc214f723762685816e592c5a8b2c9d367c33c4d39a02f5de66d5b6e322","source":{"kind":"arxiv","id":"2506.10403","version":1},"attestation_state":"computed","paper":{"title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Frederic Sala, Harit Vishwakarma, Tzu-Heng Huang","submitted_at":"2025-06-12T06:53:22Z","abstract_excerpt":"Large language models (LLMs) are widely used to evaluate the quality of LLM generations and responses, but this leads to significant challenges: high API costs, uncertain reliability, inflexible pipelines, and inherent biases. To address these, we introduce PAJAMA (Program-As-a-Judge for Automated Model Assessment), a new alternative that uses LLMs to synthesize executable judging programs instead of directly scoring responses. These synthesized programs can be stored and run locally, costing orders of magnitude less while providing interpretable, and auditable judging logic that can be easily"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2506.10403","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-12T06:53:22Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a1adbd5af7e4b6a444c32f2eadf3c6d54d1454297631292e92ef5ded3fb75b22","abstract_canon_sha256":"f63b512bace3e49f24940810e11ca66f391fc035ea6fed3fde239061a2d5fecf"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:20:22.794685Z","signature_b64":"NShC3pJ+IhnJ7guhgmvNUWPar59my9YcT7zH8I52pYTeXPAl3K25FKVbliCoNz74tfg37w8eMObjeJlSu+46CA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a2f0cfc214f723762685816e592c5a8b2c9d367c33c4d39a02f5de66d5b6e322","last_reissued_at":"2026-07-05T11:20:22.794204Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:20:22.794204Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Frederic Sala, Harit Vishwakarma, Tzu-Heng Huang","submitted_at":"2025-06-12T06:53:22Z","abstract_excerpt":"Large language models (LLMs) are widely used to evaluate the quality of LLM generations and responses, but this leads to significant challenges: high API costs, uncertain reliability, inflexible pipelines, and inherent biases. To address these, we introduce PAJAMA (Program-As-a-Judge for Automated Model Assessment), a new alternative that uses LLMs to synthesize executable judging programs instead of directly scoring responses. These synthesized programs can be stored and run locally, costing orders of magnitude less while providing interpretable, and auditable judging logic that can be easily"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.10403","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.10403/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2506.10403","created_at":"2026-07-05T11:20:22.794260+00:00"},{"alias_kind":"arxiv_version","alias_value":"2506.10403v1","created_at":"2026-07-05T11:20:22.794260+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.10403","created_at":"2026-07-05T11:20:22.794260+00:00"},{"alias_kind":"pith_short_12","alias_value":"ULYM7QQU64RX","created_at":"2026-07-05T11:20:22.794260+00:00"},{"alias_kind":"pith_short_16","alias_value":"ULYM7QQU64RXMJUF","created_at":"2026-07-05T11:20:22.794260+00:00"},{"alias_kind":"pith_short_8","alias_value":"ULYM7QQU","created_at":"2026-07-05T11:20:22.794260+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":2,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.04532","citing_title":"Multilingual Prompt Localization for Agent-as-a-Judge: Language and Backbone Sensitivity in Requirement-Level Evaluation","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2604.05083","citing_title":"Beyond LLM-as-a-Judge: Deterministic Metrics for Multilingual Generative Text Evaluation","ref_index":3,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM","json":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM.json","graph_json":"https://pith.science/api/pith-number/ULYM7QQU64RXMJUFQFXFSLC2RM/graph.json","events_json":"https://pith.science/api/pith-number/ULYM7QQU64RXMJUFQFXFSLC2RM/events.json","paper":"https://pith.science/paper/ULYM7QQU"},"agent_actions":{"view_html":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM","download_json":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM.json","view_paper":"https://pith.science/paper/ULYM7QQU","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2506.10403&json=true","fetch_graph":"https://pith.science/api/pith-number/ULYM7QQU64RXMJUFQFXFSLC2RM/graph.json","fetch_events":"https://pith.science/api/pith-number/ULYM7QQU64RXMJUFQFXFSLC2RM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/action/storage_attestation","attest_author":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/action/author_attestation","sign_citation":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/action/citation_signature","submit_replication":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/action/replication_record"}},"created_at":"2026-07-05T11:20:22.794260+00:00","updated_at":"2026-07-05T11:20:22.794260+00:00"}