{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:L62MDSBQR3MHXO5ITUMRZZUSWC","short_pith_number":"pith:L62MDSBQ","schema_version":"1.0","canonical_sha256":"5fb4c1c8308ed87bbba89d191ce692b0a8dec4c8165ad911a15eacb26d69e005","source":{"kind":"arxiv","id":"2401.04757","version":1},"attestation_state":"computed","paper":{"title":"How predictable is language model benchmark performance?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"David Owen","submitted_at":"2024-01-09T17:34:30Z","abstract_excerpt":"We investigate large language model performance across five orders of magnitude of compute scaling in eleven recent model architectures. We show that average benchmark performance, aggregating over many individual tasks and evaluations as in the commonly-used BIG-Bench dataset, is decently predictable as a function of training compute scale. Specifically, when extrapolating BIG-Bench Hard performance across one order of magnitude in compute, we observe average absolute errors of 6 percentage points (pp). By contrast, extrapolation for individual BIG-Bench tasks across an order of magnitude in "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2401.04757","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-01-09T17:34:30Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1a9f49ad719c94c40359d96569f3645cd5c3db128353a33bdee53fded19819c0","abstract_canon_sha256":"9499b5df0c702a5b796550e4b93487d0dcdd24b3e93f8a415f2696c52b08b2e0"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:31:57.598069Z","signature_b64":"04QmMHlP2etl0STOtpyhTcMKm6GbkW5L/LV7e2zIFYiXhjD4GPFowuDn2xVhGoxltAPE6Oh1SSNvVh6fca8vDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5fb4c1c8308ed87bbba89d191ce692b0a8dec4c8165ad911a15eacb26d69e005","last_reissued_at":"2026-07-05T07:31:57.597691Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:31:57.597691Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"How predictable is language model benchmark performance?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"David Owen","submitted_at":"2024-01-09T17:34:30Z","abstract_excerpt":"We investigate large language model performance across five orders of magnitude of compute scaling in eleven recent model architectures. We show that average benchmark performance, aggregating over many individual tasks and evaluations as in the commonly-used BIG-Bench dataset, is decently predictable as a function of training compute scale. Specifically, when extrapolating BIG-Bench Hard performance across one order of magnitude in compute, we observe average absolute errors of 6 percentage points (pp). By contrast, extrapolation for individual BIG-Bench tasks across an order of magnitude in "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.04757","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2401.04757/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2401.04757","created_at":"2026-07-05T07:31:57.597749+00:00"},{"alias_kind":"arxiv_version","alias_value":"2401.04757v1","created_at":"2026-07-05T07:31:57.597749+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.04757","created_at":"2026-07-05T07:31:57.597749+00:00"},{"alias_kind":"pith_short_12","alias_value":"L62MDSBQR3MH","created_at":"2026-07-05T07:31:57.597749+00:00"},{"alias_kind":"pith_short_16","alias_value":"L62MDSBQR3MHXO5I","created_at":"2026-07-05T07:31:57.597749+00:00"},{"alias_kind":"pith_short_8","alias_value":"L62MDSBQ","created_at":"2026-07-05T07:31:57.597749+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":9,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2607.00913","citing_title":"Two AI Metrics Diverged: Will it Make All the Difference?","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05029","citing_title":"Validity Threats for Foundation Model Research","ref_index":73,"is_internal_anchor":false},{"citing_arxiv_id":"2605.28179","citing_title":"SuperValid: Capability-Aligned OOD Validation for Generalizable Downstream Scaling","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10933","citing_title":"DECO: Sparse Mixture-of-Experts with Dense-Comparable Performance on End-Side Devices","ref_index":143,"is_internal_anchor":false},{"citing_arxiv_id":"2510.13786","citing_title":"The Art of Scaling Reinforcement Learning Compute for LLMs","ref_index":15,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10933","citing_title":"DECO: Sparse Mixture-of-Experts with Dense-Comparable Performance on End-Side Devices","ref_index":143,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10933","citing_title":"DECO: Sparse Mixture-of-Experts with Dense-Comparable Performance on End-Side Devices","ref_index":143,"is_internal_anchor":false},{"citing_arxiv_id":"2505.07062","citing_title":"Seed1.5-VL Technical Report","ref_index":102,"is_internal_anchor":false},{"citing_arxiv_id":"2501.14249","citing_title":"Humanity's Last Exam","ref_index":42,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC","json":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC.json","graph_json":"https://pith.science/api/pith-number/L62MDSBQR3MHXO5ITUMRZZUSWC/graph.json","events_json":"https://pith.science/api/pith-number/L62MDSBQR3MHXO5ITUMRZZUSWC/events.json","paper":"https://pith.science/paper/L62MDSBQ"},"agent_actions":{"view_html":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC","download_json":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC.json","view_paper":"https://pith.science/paper/L62MDSBQ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2401.04757&json=true","fetch_graph":"https://pith.science/api/pith-number/L62MDSBQR3MHXO5ITUMRZZUSWC/graph.json","fetch_events":"https://pith.science/api/pith-number/L62MDSBQR3MHXO5ITUMRZZUSWC/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC/action/timestamp_anchor","attest_storage":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC/action/storage_attestation","attest_author":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC/action/author_attestation","sign_citation":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC/action/citation_signature","submit_replication":"https://pith.science/pith/L62MDSBQR3MHXO5ITUMRZZUSWC/action/replication_record"}},"created_at":"2026-07-05T07:31:57.597749+00:00","updated_at":"2026-07-05T07:31:57.597749+00:00"}