{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:ULYM7QQU64RXMJUFQFXFSLC2RM","short_pith_number":"pith:ULYM7QQU","canonical_record":{"source":{"id":"2506.10403","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-12T06:53:22Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a1adbd5af7e4b6a444c32f2eadf3c6d54d1454297631292e92ef5ded3fb75b22","abstract_canon_sha256":"f63b512bace3e49f24940810e11ca66f391fc035ea6fed3fde239061a2d5fecf"},"schema_version":"1.0"},"canonical_sha256":"a2f0cfc214f723762685816e592c5a8b2c9d367c33c4d39a02f5de66d5b6e322","source":{"kind":"arxiv","id":"2506.10403","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.10403","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"arxiv_version","alias_value":"2506.10403v1","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.10403","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"pith_short_12","alias_value":"ULYM7QQU64RX","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"pith_short_16","alias_value":"ULYM7QQU64RXMJUF","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"pith_short_8","alias_value":"ULYM7QQU","created_at":"2026-07-05T11:20:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:ULYM7QQU64RXMJUFQFXFSLC2RM","target":"record","payload":{"canonical_record":{"source":{"id":"2506.10403","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-12T06:53:22Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a1adbd5af7e4b6a444c32f2eadf3c6d54d1454297631292e92ef5ded3fb75b22","abstract_canon_sha256":"f63b512bace3e49f24940810e11ca66f391fc035ea6fed3fde239061a2d5fecf"},"schema_version":"1.0"},"canonical_sha256":"a2f0cfc214f723762685816e592c5a8b2c9d367c33c4d39a02f5de66d5b6e322","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:20:22.794685Z","signature_b64":"NShC3pJ+IhnJ7guhgmvNUWPar59my9YcT7zH8I52pYTeXPAl3K25FKVbliCoNz74tfg37w8eMObjeJlSu+46CA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a2f0cfc214f723762685816e592c5a8b2c9d367c33c4d39a02f5de66d5b6e322","last_reissued_at":"2026-07-05T11:20:22.794204Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:20:22.794204Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.10403","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:20:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"07sJy/lvp4rVngaf4ilFTEf1kALYAf2vVYGNkYYFE0y+htbkbDwP2XbxY5B1EBPiX0Bc8gQf5cJA2lT/J4X/Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T04:54:42.345922Z"},"content_sha256":"aa245fb4944db92dc71a821f6279c85782c72d7f009b25c6bbb208f774f03b62","schema_version":"1.0","event_id":"sha256:aa245fb4944db92dc71a821f6279c85782c72d7f009b25c6bbb208f774f03b62"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:ULYM7QQU64RXMJUFQFXFSLC2RM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Frederic Sala, Harit Vishwakarma, Tzu-Heng Huang","submitted_at":"2025-06-12T06:53:22Z","abstract_excerpt":"Large language models (LLMs) are widely used to evaluate the quality of LLM generations and responses, but this leads to significant challenges: high API costs, uncertain reliability, inflexible pipelines, and inherent biases. To address these, we introduce PAJAMA (Program-As-a-Judge for Automated Model Assessment), a new alternative that uses LLMs to synthesize executable judging programs instead of directly scoring responses. These synthesized programs can be stored and run locally, costing orders of magnitude less while providing interpretable, and auditable judging logic that can be easily"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.10403","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.10403/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:20:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"B77o1xaDUZjna+QDozQNs+o18HD2/qt25RRbUWOzr2bujfusvtdmiXAtDV26bDI0/7dpLT7y56QUwp/j2DlIDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T04:54:42.346516Z"},"content_sha256":"1a6a1315af561d2bf99ad3d492107a8b870c696185674566a6571987b7f244b0","schema_version":"1.0","event_id":"sha256:1a6a1315af561d2bf99ad3d492107a8b870c696185674566a6571987b7f244b0"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/bundle.json","state_url":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T04:54:42Z","links":{"resolver":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM","bundle":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/bundle.json","state":"https://pith.science/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ULYM7QQU64RXMJUFQFXFSLC2RM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:ULYM7QQU64RXMJUFQFXFSLC2RM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f63b512bace3e49f24940810e11ca66f391fc035ea6fed3fde239061a2d5fecf","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-12T06:53:22Z","title_canon_sha256":"a1adbd5af7e4b6a444c32f2eadf3c6d54d1454297631292e92ef5ded3fb75b22"},"schema_version":"1.0","source":{"id":"2506.10403","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.10403","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"arxiv_version","alias_value":"2506.10403v1","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.10403","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"pith_short_12","alias_value":"ULYM7QQU64RX","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"pith_short_16","alias_value":"ULYM7QQU64RXMJUF","created_at":"2026-07-05T11:20:22Z"},{"alias_kind":"pith_short_8","alias_value":"ULYM7QQU","created_at":"2026-07-05T11:20:22Z"}],"graph_snapshots":[{"event_id":"sha256:1a6a1315af561d2bf99ad3d492107a8b870c696185674566a6571987b7f244b0","target":"graph","created_at":"2026-07-05T11:20:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.10403/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) are widely used to evaluate the quality of LLM generations and responses, but this leads to significant challenges: high API costs, uncertain reliability, inflexible pipelines, and inherent biases. To address these, we introduce PAJAMA (Program-As-a-Judge for Automated Model Assessment), a new alternative that uses LLMs to synthesize executable judging programs instead of directly scoring responses. These synthesized programs can be stored and run locally, costing orders of magnitude less while providing interpretable, and auditable judging logic that can be easily","authors_text":"Frederic Sala, Harit Vishwakarma, Tzu-Heng Huang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-12T06:53:22Z","title":"Time To Impeach LLM-as-a-Judge: Programs are the Future of Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.10403","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:aa245fb4944db92dc71a821f6279c85782c72d7f009b25c6bbb208f774f03b62","target":"record","created_at":"2026-07-05T11:20:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f63b512bace3e49f24940810e11ca66f391fc035ea6fed3fde239061a2d5fecf","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2025-06-12T06:53:22Z","title_canon_sha256":"a1adbd5af7e4b6a444c32f2eadf3c6d54d1454297631292e92ef5ded3fb75b22"},"schema_version":"1.0","source":{"id":"2506.10403","kind":"arxiv","version":1}},"canonical_sha256":"a2f0cfc214f723762685816e592c5a8b2c9d367c33c4d39a02f5de66d5b6e322","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a2f0cfc214f723762685816e592c5a8b2c9d367c33c4d39a02f5de66d5b6e322","first_computed_at":"2026-07-05T11:20:22.794204Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:20:22.794204Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NShC3pJ+IhnJ7guhgmvNUWPar59my9YcT7zH8I52pYTeXPAl3K25FKVbliCoNz74tfg37w8eMObjeJlSu+46CA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:20:22.794685Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.10403","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:aa245fb4944db92dc71a821f6279c85782c72d7f009b25c6bbb208f774f03b62","sha256:1a6a1315af561d2bf99ad3d492107a8b870c696185674566a6571987b7f244b0"],"state_sha256":"c114b39682d2fcd946afd9bff56df4ada2c0a02e530bec6d2150ddf31a4d4325"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"u1Azjg13laPymNiG3yEroYtEtu011v99qGUKJfu7Bhso2nhhzKvBkL+vU/PeiAEVJxSFetg6nadMY/Q5yB0uAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T04:54:42.350670Z","bundle_sha256":"c83ed32b3df81158ed32901f339c8f3b1a915283be5a71d2b1c270e0c454b46e"}}