{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:R2R7U6PQDVZ5DOTL4W5K6EMKN7","short_pith_number":"pith:R2R7U6PQ","canonical_record":{"source":{"id":"2403.12031","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-18T17:59:04Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c7fc028c8c3dbff34ae1bd9f8a30b5c54291e07cb78f6951b382744d222528bd","abstract_canon_sha256":"bda8039c94f4424c5e347f0074762cab29d9678cf0818500f158041527c5c654"},"schema_version":"1.0"},"canonical_sha256":"8ea3fa79f01d73d1ba6be5baaf118a6ff00aa633fb76013072f0dec967846623","source":{"kind":"arxiv","id":"2403.12031","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.12031","created_at":"2026-05-17T23:38:48Z"},{"alias_kind":"arxiv_version","alias_value":"2403.12031v2","created_at":"2026-05-17T23:38:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.12031","created_at":"2026-05-17T23:38:48Z"},{"alias_kind":"pith_short_12","alias_value":"R2R7U6PQDVZ5","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"R2R7U6PQDVZ5DOTL","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"R2R7U6PQ","created_at":"2026-05-18T12:33:37Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:R2R7U6PQDVZ5DOTL4W5K6EMKN7","target":"record","payload":{"canonical_record":{"source":{"id":"2403.12031","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-18T17:59:04Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"c7fc028c8c3dbff34ae1bd9f8a30b5c54291e07cb78f6951b382744d222528bd","abstract_canon_sha256":"bda8039c94f4424c5e347f0074762cab29d9678cf0818500f158041527c5c654"},"schema_version":"1.0"},"canonical_sha256":"8ea3fa79f01d73d1ba6be5baaf118a6ff00aa633fb76013072f0dec967846623","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:48.141231Z","signature_b64":"qTHO8m9dgiOp+qB6UkQIS9HWSBfc5D0qD02e2PjzLXRVRGZkv4r3TUZ+g3YERIPZ5mqWyWeBujewBosajU5BBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8ea3fa79f01d73d1ba6be5baaf118a6ff00aa633fb76013072f0dec967846623","last_reissued_at":"2026-05-17T23:38:48.140645Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:48.140645Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2403.12031","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DfOkVEFTJNQ5gaRgGPZnjGpGk5SHfGuEOwoKO8Mbu01DdADl9//e98KVOAseOQW0qok2dO/2rKkTe9Th0/W+Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-27T03:52:47.754222Z"},"content_sha256":"189ef7a6b91f3b0e432bbe7ecc6e492e1c0cb9060fc0e93987e899d1e9f32dce","schema_version":"1.0","event_id":"sha256:189ef7a6b91f3b0e432bbe7ecc6e492e1c0cb9060fc0e93987e899d1e9f32dce"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:R2R7U6PQDVZ5DOTL4W5K6EMKN7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"RouterBench: A Benchmark for Multi-LLM Routing System","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"RouterBench supplies a benchmark and over 405k inference results to evaluate systems that route queries across multiple LLMs.","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Benjamin Keigwin, Gaurav Ranganath, Jacob Bieker, Kurt Keutzer, Nan Jiang, Qitian Jason Hu, Shriyash Kaustubh Upadhyay, Xiuyu Li","submitted_at":"2024-03-18T17:59:04Z","abstract_excerpt":"As the range of applications for Large Language Models (LLMs) continues to grow, the demand for effective serving solutions becomes increasingly critical. Despite the versatility of LLMs, no single model can optimally address all tasks and applications, particularly when balancing performance with cost. This limitation has led to the development of LLM routing systems, which combine the strengths of various models to overcome the constraints of individual LLMs. Yet, the absence of a standardized benchmark for evaluating the performance of LLM routers hinders progress in this area. To bridge th"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"We present RouterBench, a novel evaluation framework designed to systematically assess the efficacy of LLM routing systems, along with a comprehensive dataset comprising over 405k inference outcomes from representative LLMs to support the development of routing strategies.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"The chosen tasks, models, and inference outcomes in the 405k dataset are sufficiently representative of real-world usage patterns and future models to make the benchmark generalizable.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"RouterBench supplies a standardized benchmark, 405k+ inference dataset, theoretical framework, and comparative analysis for multi-LLM routing systems.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"RouterBench supplies a benchmark and over 405k inference results to evaluate systems that route queries across multiple LLMs.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"65c51edcf555b3810acd52be277621715afc3535c789b593fffe6a3073315abe"},"source":{"id":"2403.12031","kind":"arxiv","version":2},"verdict":{"id":"c0a1f036-fac3-4bc0-85a6-ca4041050dcb","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-16T10:43:52.460809Z","strongest_claim":"We present RouterBench, a novel evaluation framework designed to systematically assess the efficacy of LLM routing systems, along with a comprehensive dataset comprising over 405k inference outcomes from representative LLMs to support the development of routing strategies.","one_line_summary":"RouterBench supplies a standardized benchmark, 405k+ inference dataset, theoretical framework, and comparative analysis for multi-LLM routing systems.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"The chosen tasks, models, and inference outcomes in the 405k dataset are sufficiently representative of real-world usage patterns and future models to make the benchmark generalizable.","pith_extraction_headline":"RouterBench supplies a benchmark and over 405k inference results to evaluate systems that route queries across multiple LLMs."},"references":{"count":110,"sample":[{"doi":"","year":2024,"title":"2024 , journal =","work_id":"1cef5e88-d6f5-4fef-b8a9-15e89f3ebe83","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"The Twelfth International Conference on Learning Representations , year=","work_id":"8299efba-a1bc-46f2-939b-57c0bc277e6c","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","work_id":"ed240a10-5b19-406c-baa5-30803f465785","ref_index":4,"cited_arxiv_id":"1810.04805","is_internal_anchor":true},{"doi":"","year":2000,"title":"Langley , title =","work_id":"6cd283dc-0548-45e9-af07-6bc1005593ad","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":1980,"title":"T. M. Mitchell. The Need for Biases in Learning Generalizations. 1980","work_id":"6b09bca6-ef5d-4a83-8c8e-219f23cbd761","ref_index":6,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":110,"snapshot_sha256":"d58bb57c5da0012c18249ad284b083c1ce58a7c131778c95df4c5321b7bc21ed","internal_anchors":16},"formal_canon":{"evidence_count":2,"snapshot_sha256":"9e85b8316895179fdbed5bb389d11b9ac1d93fa7e95c70761bf3fc7d95be2d2a"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"c0a1f036-fac3-4bc0-85a6-ca4041050dcb"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"B/4KnRn5VcoFHA07mt34WKeIUg99QDxNOpZky6L2mpY01BfZmvgoybz1COLzjDe9uGVIiZduuZyiKRPoKx7YDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-27T03:52:47.755216Z"},"content_sha256":"6a5db8e3028ce41eaaea149c34bf48c35ada7d3e111b7a5763927536c6b60fea","schema_version":"1.0","event_id":"sha256:6a5db8e3028ce41eaaea149c34bf48c35ada7d3e111b7a5763927536c6b60fea"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/R2R7U6PQDVZ5DOTL4W5K6EMKN7/bundle.json","state_url":"https://pith.science/pith/R2R7U6PQDVZ5DOTL4W5K6EMKN7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/R2R7U6PQDVZ5DOTL4W5K6EMKN7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-27T03:52:47Z","links":{"resolver":"https://pith.science/pith/R2R7U6PQDVZ5DOTL4W5K6EMKN7","bundle":"https://pith.science/pith/R2R7U6PQDVZ5DOTL4W5K6EMKN7/bundle.json","state":"https://pith.science/pith/R2R7U6PQDVZ5DOTL4W5K6EMKN7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/R2R7U6PQDVZ5DOTL4W5K6EMKN7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:R2R7U6PQDVZ5DOTL4W5K6EMKN7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bda8039c94f4424c5e347f0074762cab29d9678cf0818500f158041527c5c654","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-18T17:59:04Z","title_canon_sha256":"c7fc028c8c3dbff34ae1bd9f8a30b5c54291e07cb78f6951b382744d222528bd"},"schema_version":"1.0","source":{"id":"2403.12031","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.12031","created_at":"2026-05-17T23:38:48Z"},{"alias_kind":"arxiv_version","alias_value":"2403.12031v2","created_at":"2026-05-17T23:38:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.12031","created_at":"2026-05-17T23:38:48Z"},{"alias_kind":"pith_short_12","alias_value":"R2R7U6PQDVZ5","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_16","alias_value":"R2R7U6PQDVZ5DOTL","created_at":"2026-05-18T12:33:37Z"},{"alias_kind":"pith_short_8","alias_value":"R2R7U6PQ","created_at":"2026-05-18T12:33:37Z"}],"graph_snapshots":[{"event_id":"sha256:6a5db8e3028ce41eaaea149c34bf48c35ada7d3e111b7a5763927536c6b60fea","target":"graph","created_at":"2026-05-17T23:38:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"We present RouterBench, a novel evaluation framework designed to systematically assess the efficacy of LLM routing systems, along with a comprehensive dataset comprising over 405k inference outcomes from representative LLMs to support the development of routing strategies."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"The chosen tasks, models, and inference outcomes in the 405k dataset are sufficiently representative of real-world usage patterns and future models to make the benchmark generalizable."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"RouterBench supplies a standardized benchmark, 405k+ inference dataset, theoretical framework, and comparative analysis for multi-LLM routing systems."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"RouterBench supplies a benchmark and over 405k inference results to evaluate systems that route queries across multiple LLMs."}],"snapshot_sha256":"65c51edcf555b3810acd52be277621715afc3535c789b593fffe6a3073315abe"},"formal_canon":{"evidence_count":2,"snapshot_sha256":"9e85b8316895179fdbed5bb389d11b9ac1d93fa7e95c70761bf3fc7d95be2d2a"},"paper":{"abstract_excerpt":"As the range of applications for Large Language Models (LLMs) continues to grow, the demand for effective serving solutions becomes increasingly critical. Despite the versatility of LLMs, no single model can optimally address all tasks and applications, particularly when balancing performance with cost. This limitation has led to the development of LLM routing systems, which combine the strengths of various models to overcome the constraints of individual LLMs. Yet, the absence of a standardized benchmark for evaluating the performance of LLM routers hinders progress in this area. To bridge th","authors_text":"Benjamin Keigwin, Gaurav Ranganath, Jacob Bieker, Kurt Keutzer, Nan Jiang, Qitian Jason Hu, Shriyash Kaustubh Upadhyay, Xiuyu Li","cross_cats":["cs.AI"],"headline":"RouterBench supplies a benchmark and over 405k inference results to evaluate systems that route queries across multiple LLMs.","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-18T17:59:04Z","title":"RouterBench: A Benchmark for Multi-LLM Routing System"},"references":{"count":110,"internal_anchors":16,"resolved_work":110,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"2024 , journal =","work_id":"1cef5e88-d6f5-4fef-b8a9-15e89f3ebe83","year":2024},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"The Twelfth International Conference on Learning Representations , year=","work_id":"8299efba-a1bc-46f2-939b-57c0bc277e6c","year":null},{"cited_arxiv_id":"1810.04805","doi":"","is_internal_anchor":true,"ref_index":4,"title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","work_id":"ed240a10-5b19-406c-baa5-30803f465785","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"Langley , title =","work_id":"6cd283dc-0548-45e9-af07-6bc1005593ad","year":2000},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":6,"title":"T. M. Mitchell. The Need for Biases in Learning Generalizations. 1980","work_id":"6b09bca6-ef5d-4a83-8c8e-219f23cbd761","year":1980}],"snapshot_sha256":"d58bb57c5da0012c18249ad284b083c1ce58a7c131778c95df4c5321b7bc21ed"},"source":{"id":"2403.12031","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-16T10:43:52.460809Z","id":"c0a1f036-fac3-4bc0-85a6-ca4041050dcb","model_set":{"reader":"grok-4.3"},"one_line_summary":"RouterBench supplies a standardized benchmark, 405k+ inference dataset, theoretical framework, and comparative analysis for multi-LLM routing systems.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"RouterBench supplies a benchmark and over 405k inference results to evaluate systems that route queries across multiple LLMs.","strongest_claim":"We present RouterBench, a novel evaluation framework designed to systematically assess the efficacy of LLM routing systems, along with a comprehensive dataset comprising over 405k inference outcomes from representative LLMs to support the development of routing strategies.","weakest_assumption":"The chosen tasks, models, and inference outcomes in the 405k dataset are sufficiently representative of real-world usage patterns and future models to make the benchmark generalizable."}},"verdict_id":"c0a1f036-fac3-4bc0-85a6-ca4041050dcb"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:189ef7a6b91f3b0e432bbe7ecc6e492e1c0cb9060fc0e93987e899d1e9f32dce","target":"record","created_at":"2026-05-17T23:38:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bda8039c94f4424c5e347f0074762cab29d9678cf0818500f158041527c5c654","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-03-18T17:59:04Z","title_canon_sha256":"c7fc028c8c3dbff34ae1bd9f8a30b5c54291e07cb78f6951b382744d222528bd"},"schema_version":"1.0","source":{"id":"2403.12031","kind":"arxiv","version":2}},"canonical_sha256":"8ea3fa79f01d73d1ba6be5baaf118a6ff00aa633fb76013072f0dec967846623","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8ea3fa79f01d73d1ba6be5baaf118a6ff00aa633fb76013072f0dec967846623","first_computed_at":"2026-05-17T23:38:48.140645Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:48.140645Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"qTHO8m9dgiOp+qB6UkQIS9HWSBfc5D0qD02e2PjzLXRVRGZkv4r3TUZ+g3YERIPZ5mqWyWeBujewBosajU5BBw==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:48.141231Z","signed_message":"canonical_sha256_bytes"},"source_id":"2403.12031","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:189ef7a6b91f3b0e432bbe7ecc6e492e1c0cb9060fc0e93987e899d1e9f32dce","sha256:6a5db8e3028ce41eaaea149c34bf48c35ada7d3e111b7a5763927536c6b60fea"],"state_sha256":"e79408aef419c70cf0641c132eb639ec0ee9ddd10dd7e82300eada7a409f919b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eUyuaPtAOubOVjYIgTTdk0MdV3nXGSolpIfZw/osxr6LLi5sx9T6J1hGu2KNkN1/Vrd5/J1q1NG4rhA3INhVDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-27T03:52:47.759582Z","bundle_sha256":"ba0bb699ae16a0ff39fd1764343ff38fc59f72cec24d6abd9c2e72e466cf7de9"}}