{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:XXELHF4UDKTNTXEIHVXJEMJIIG","short_pith_number":"pith:XXELHF4U","canonical_record":{"source":{"id":"2407.18370","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T20:04:59Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"0aa1e295eaf950db1527bcf91dabebdcab04fb9ee731ce9564f6221298a54d4a","abstract_canon_sha256":"0f87a023cf542f816ade3168b8aa8d4e9ef3a53e4c4116b3db72d01f3627dde8"},"schema_version":"1.0"},"canonical_sha256":"bdc8b397941aa6d9dc883d6e92312841891322783bed465c45bee14210ce4340","source":{"kind":"arxiv","id":"2407.18370","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.18370","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"arxiv_version","alias_value":"2407.18370v1","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.18370","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"pith_short_12","alias_value":"XXELHF4UDKTN","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"pith_short_16","alias_value":"XXELHF4UDKTNTXEI","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"pith_short_8","alias_value":"XXELHF4U","created_at":"2026-07-05T08:48:48Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:XXELHF4UDKTNTXEIHVXJEMJIIG","target":"record","payload":{"canonical_record":{"source":{"id":"2407.18370","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T20:04:59Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"0aa1e295eaf950db1527bcf91dabebdcab04fb9ee731ce9564f6221298a54d4a","abstract_canon_sha256":"0f87a023cf542f816ade3168b8aa8d4e9ef3a53e4c4116b3db72d01f3627dde8"},"schema_version":"1.0"},"canonical_sha256":"bdc8b397941aa6d9dc883d6e92312841891322783bed465c45bee14210ce4340","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:48:48.697372Z","signature_b64":"gyOb9dF/hP/TDm7strN0FHsI6R1oeT4VPWgmetqJy1JduayEb3muEVvIvsRmi2QNpcdTTwgRF2tjYV13RnO7Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"bdc8b397941aa6d9dc883d6e92312841891322783bed465c45bee14210ce4340","last_reissued_at":"2026-07-05T08:48:48.696898Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:48:48.696898Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.18370","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:48:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"bgzYMImxos0F9fMDctMkr1mtiRLyqmMy4SK/OruF4FNvfG7Atzo7eWOCpu8lSnF3ngv2i7OIWLpYF6WjpzahAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T07:05:53.615156Z"},"content_sha256":"161ba2172d8a18fe7ad455e10c0aa0d35e4d597e7f381fd752bcdbe72051f164","schema_version":"1.0","event_id":"sha256:161ba2172d8a18fe7ad455e10c0aa0d35e4d597e7f381fd752bcdbe72051f164"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:XXELHF4UDKTNTXEIHVXJEMJIIG","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Trust or Escalate: LLM Judges with Provable Guarantees for Human Agreement","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.LG","authors_text":"Faeze Brahman, Jaehun Jung, Yejin Choi","submitted_at":"2024-07-25T20:04:59Z","abstract_excerpt":"We present a principled approach to provide LLM-based evaluation with a rigorous guarantee of human agreement. We first propose that a reliable evaluation method should not uncritically rely on model preferences for pairwise evaluation, but rather assess the confidence of judge models and selectively decide when to trust its judgement. We then show that under this selective evaluation framework, human agreement can be provably guaranteed -- such that the model evaluation aligns with that of humans to a user-specified agreement level. As part of our framework, we also introduce Simulated Annota"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.18370","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.18370/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:48:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"9+ipALGnwAQCBLlYdQgFzpXYBK6+M2G7sEx4eDKJSAy/GEoUupCpE9CnYo9g5kI7cbQuKksckOKSBZMdRuIXAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T07:05:53.615503Z"},"content_sha256":"0382e8327d8fd326fffd84adfcb8ea5481021ae53888d425cfc3d4631eb0229a","schema_version":"1.0","event_id":"sha256:0382e8327d8fd326fffd84adfcb8ea5481021ae53888d425cfc3d4631eb0229a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/XXELHF4UDKTNTXEIHVXJEMJIIG/bundle.json","state_url":"https://pith.science/pith/XXELHF4UDKTNTXEIHVXJEMJIIG/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/XXELHF4UDKTNTXEIHVXJEMJIIG/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T07:05:53Z","links":{"resolver":"https://pith.science/pith/XXELHF4UDKTNTXEIHVXJEMJIIG","bundle":"https://pith.science/pith/XXELHF4UDKTNTXEIHVXJEMJIIG/bundle.json","state":"https://pith.science/pith/XXELHF4UDKTNTXEIHVXJEMJIIG/state.json","well_known_bundle":"https://pith.science/.well-known/pith/XXELHF4UDKTNTXEIHVXJEMJIIG/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:XXELHF4UDKTNTXEIHVXJEMJIIG","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0f87a023cf542f816ade3168b8aa8d4e9ef3a53e4c4116b3db72d01f3627dde8","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T20:04:59Z","title_canon_sha256":"0aa1e295eaf950db1527bcf91dabebdcab04fb9ee731ce9564f6221298a54d4a"},"schema_version":"1.0","source":{"id":"2407.18370","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.18370","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"arxiv_version","alias_value":"2407.18370v1","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.18370","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"pith_short_12","alias_value":"XXELHF4UDKTN","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"pith_short_16","alias_value":"XXELHF4UDKTNTXEI","created_at":"2026-07-05T08:48:48Z"},{"alias_kind":"pith_short_8","alias_value":"XXELHF4U","created_at":"2026-07-05T08:48:48Z"}],"graph_snapshots":[{"event_id":"sha256:0382e8327d8fd326fffd84adfcb8ea5481021ae53888d425cfc3d4631eb0229a","target":"graph","created_at":"2026-07-05T08:48:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.18370/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We present a principled approach to provide LLM-based evaluation with a rigorous guarantee of human agreement. We first propose that a reliable evaluation method should not uncritically rely on model preferences for pairwise evaluation, but rather assess the confidence of judge models and selectively decide when to trust its judgement. We then show that under this selective evaluation framework, human agreement can be provably guaranteed -- such that the model evaluation aligns with that of humans to a user-specified agreement level. As part of our framework, we also introduce Simulated Annota","authors_text":"Faeze Brahman, Jaehun Jung, Yejin Choi","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T20:04:59Z","title":"Trust or Escalate: LLM Judges with Provable Guarantees for Human Agreement"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.18370","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:161ba2172d8a18fe7ad455e10c0aa0d35e4d597e7f381fd752bcdbe72051f164","target":"record","created_at":"2026-07-05T08:48:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0f87a023cf542f816ade3168b8aa8d4e9ef3a53e4c4116b3db72d01f3627dde8","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-25T20:04:59Z","title_canon_sha256":"0aa1e295eaf950db1527bcf91dabebdcab04fb9ee731ce9564f6221298a54d4a"},"schema_version":"1.0","source":{"id":"2407.18370","kind":"arxiv","version":1}},"canonical_sha256":"bdc8b397941aa6d9dc883d6e92312841891322783bed465c45bee14210ce4340","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"bdc8b397941aa6d9dc883d6e92312841891322783bed465c45bee14210ce4340","first_computed_at":"2026-07-05T08:48:48.696898Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:48:48.696898Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gyOb9dF/hP/TDm7strN0FHsI6R1oeT4VPWgmetqJy1JduayEb3muEVvIvsRmi2QNpcdTTwgRF2tjYV13RnO7Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:48:48.697372Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.18370","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:161ba2172d8a18fe7ad455e10c0aa0d35e4d597e7f381fd752bcdbe72051f164","sha256:0382e8327d8fd326fffd84adfcb8ea5481021ae53888d425cfc3d4631eb0229a"],"state_sha256":"cf228fcc379fae23487fdd580c7e8095f7586382ddbcfc145c6424d653930847"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KkIZVz951GA+MktfOvZ61Gb3LPtsfMDKp+YX/oAeh9oB5VTFV/1gT5z17nQtMzzHx17OIFB45nA56kcqe6k9Bg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T07:05:53.618651Z","bundle_sha256":"ce6ede53175fcbc58286d84f3ea475337e37f2bca7d581ee9543e8deecd7526b"}}