{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:IM7V7O23YQLMWWBMROSTUKQ4XL","short_pith_number":"pith:IM7V7O23","canonical_record":{"source":{"id":"2410.13284","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T07:28:18Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"2f9a3f5506b4ea9c3373bcbb8f4a1e4e4860f00cbeaacbea3914fd17a0bb4d08","abstract_canon_sha256":"0aa48f1b77d96826e89bded76d6d1a9044521cbbe6d6722e3a71e953d7c714a9"},"schema_version":"1.0"},"canonical_sha256":"433f5fbb5bc416cb582c8ba53a2a1cbafb763db766d1d8eab936d15042ef20bb","source":{"kind":"arxiv","id":"2410.13284","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.13284","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"arxiv_version","alias_value":"2410.13284v3","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.13284","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"pith_short_12","alias_value":"IM7V7O23YQLM","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"pith_short_16","alias_value":"IM7V7O23YQLMWWBM","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"pith_short_8","alias_value":"IM7V7O23","created_at":"2026-07-05T11:24:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:IM7V7O23YQLMWWBMROSTUKQ4XL","target":"record","payload":{"canonical_record":{"source":{"id":"2410.13284","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T07:28:18Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"2f9a3f5506b4ea9c3373bcbb8f4a1e4e4860f00cbeaacbea3914fd17a0bb4d08","abstract_canon_sha256":"0aa48f1b77d96826e89bded76d6d1a9044521cbbe6d6722e3a71e953d7c714a9"},"schema_version":"1.0"},"canonical_sha256":"433f5fbb5bc416cb582c8ba53a2a1cbafb763db766d1d8eab936d15042ef20bb","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:24:07.332994Z","signature_b64":"UuGczgjqpZ8unWD559k7KAJBDQNjE5nwVhMosrM5LhtDVmtNruE+TG86FGnJFkkUX7LnPj7iqg5E+ud2PylpBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"433f5fbb5bc416cb582c8ba53a2a1cbafb763db766d1d8eab936d15042ef20bb","last_reissued_at":"2026-07-05T11:24:07.332424Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:24:07.332424Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.13284","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:24:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GxIyWn43xKW6PGYnO9aN5Uf10sAp88X5joAlnpZHnFhxIPF507wdjERwwTo9ohV8pQkYFEsmKcsYHDEM0M/rBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T04:16:40.918447Z"},"content_sha256":"fbcd09336085aa647d8832e5c87c370e88e556517ed1ef0090c239f68c5bf7c2","schema_version":"1.0","event_id":"sha256:fbcd09336085aa647d8832e5c87c370e88e556517ed1ef0090c239f68c5bf7c2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:IM7V7O23YQLMWWBMROSTUKQ4XL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Learning to Route LLMs with Confidence Tokens","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Helen Zhou, John Boccio, Parikshit Gopalan, Prathusha Kameswara Sarma, Sara Bolouki, Xia Hu, Yu-Neng Chuang","submitted_at":"2024-10-17T07:28:18Z","abstract_excerpt":"Large language models (LLMs) have demonstrated impressive performance on several tasks and are increasingly deployed in real-world applications. However, especially in high-stakes settings, it becomes vital to know when the output of an LLM may be unreliable. Depending on whether an answer is trustworthy, a system can then choose to route the question to another expert, or otherwise fall back on a safe default behavior. In this work, we study the extent to which LLMs can reliably indicate confidence in their answers, and how this notion of confidence can translate into downstream accuracy gain"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.13284","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.13284/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:24:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4jcH//a9gDNAFzr4nYQfrmsQL4nccKs9XklE6RphjxHEskM3Z2i6Z9+NIQbtVJR73dOn9xJjmUPacqVYZiw6CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T04:16:40.918825Z"},"content_sha256":"db8efd0577c19ca918f1fda40ac4d3fc1cd8bd278f8e71bbc4569bdcd6ccfa2c","schema_version":"1.0","event_id":"sha256:db8efd0577c19ca918f1fda40ac4d3fc1cd8bd278f8e71bbc4569bdcd6ccfa2c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IM7V7O23YQLMWWBMROSTUKQ4XL/bundle.json","state_url":"https://pith.science/pith/IM7V7O23YQLMWWBMROSTUKQ4XL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IM7V7O23YQLMWWBMROSTUKQ4XL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T04:16:40Z","links":{"resolver":"https://pith.science/pith/IM7V7O23YQLMWWBMROSTUKQ4XL","bundle":"https://pith.science/pith/IM7V7O23YQLMWWBMROSTUKQ4XL/bundle.json","state":"https://pith.science/pith/IM7V7O23YQLMWWBMROSTUKQ4XL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IM7V7O23YQLMWWBMROSTUKQ4XL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:IM7V7O23YQLMWWBMROSTUKQ4XL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0aa48f1b77d96826e89bded76d6d1a9044521cbbe6d6722e3a71e953d7c714a9","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T07:28:18Z","title_canon_sha256":"2f9a3f5506b4ea9c3373bcbb8f4a1e4e4860f00cbeaacbea3914fd17a0bb4d08"},"schema_version":"1.0","source":{"id":"2410.13284","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.13284","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"arxiv_version","alias_value":"2410.13284v3","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.13284","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"pith_short_12","alias_value":"IM7V7O23YQLM","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"pith_short_16","alias_value":"IM7V7O23YQLMWWBM","created_at":"2026-07-05T11:24:07Z"},{"alias_kind":"pith_short_8","alias_value":"IM7V7O23","created_at":"2026-07-05T11:24:07Z"}],"graph_snapshots":[{"event_id":"sha256:db8efd0577c19ca918f1fda40ac4d3fc1cd8bd278f8e71bbc4569bdcd6ccfa2c","target":"graph","created_at":"2026-07-05T11:24:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.13284/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) have demonstrated impressive performance on several tasks and are increasingly deployed in real-world applications. However, especially in high-stakes settings, it becomes vital to know when the output of an LLM may be unreliable. Depending on whether an answer is trustworthy, a system can then choose to route the question to another expert, or otherwise fall back on a safe default behavior. In this work, we study the extent to which LLMs can reliably indicate confidence in their answers, and how this notion of confidence can translate into downstream accuracy gain","authors_text":"Helen Zhou, John Boccio, Parikshit Gopalan, Prathusha Kameswara Sarma, Sara Bolouki, Xia Hu, Yu-Neng Chuang","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T07:28:18Z","title":"Learning to Route LLMs with Confidence Tokens"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.13284","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fbcd09336085aa647d8832e5c87c370e88e556517ed1ef0090c239f68c5bf7c2","target":"record","created_at":"2026-07-05T11:24:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0aa48f1b77d96826e89bded76d6d1a9044521cbbe6d6722e3a71e953d7c714a9","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-10-17T07:28:18Z","title_canon_sha256":"2f9a3f5506b4ea9c3373bcbb8f4a1e4e4860f00cbeaacbea3914fd17a0bb4d08"},"schema_version":"1.0","source":{"id":"2410.13284","kind":"arxiv","version":3}},"canonical_sha256":"433f5fbb5bc416cb582c8ba53a2a1cbafb763db766d1d8eab936d15042ef20bb","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"433f5fbb5bc416cb582c8ba53a2a1cbafb763db766d1d8eab936d15042ef20bb","first_computed_at":"2026-07-05T11:24:07.332424Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:24:07.332424Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"UuGczgjqpZ8unWD559k7KAJBDQNjE5nwVhMosrM5LhtDVmtNruE+TG86FGnJFkkUX7LnPj7iqg5E+ud2PylpBg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:24:07.332994Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.13284","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fbcd09336085aa647d8832e5c87c370e88e556517ed1ef0090c239f68c5bf7c2","sha256:db8efd0577c19ca918f1fda40ac4d3fc1cd8bd278f8e71bbc4569bdcd6ccfa2c"],"state_sha256":"93682aad9ce5c49ddf0594a8c850691558fd677f85f3b99b0000e639b0e3cccf"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"85lVpmcI/cNmZYA7zAfMi7Vyxg4mEAmkNgMmsM7ajRo5OHhDV2b2ASwRYY3U9qd4a3xW8Nwjjt+Zwz94FRGkCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T04:16:40.920796Z","bundle_sha256":"4e08d97f45a25e24e3c287f5d6b3b0b29d292f84381612811d609bac2f56080d"}}