{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:6VXOIS6TOMFRAFMQJRO5XJ3ZEK","short_pith_number":"pith:6VXOIS6T","canonical_record":{"source":{"id":"2407.21787","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-31T17:57:25Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"31f0dd143626a168bc507ded074aa4847baab69be95bca12326b9be6945ff850","abstract_canon_sha256":"2ea7eaa9bcea62249d410bfcd9d73e3b07ae9655cf37510b4beaa4da2daecd5b"},"schema_version":"1.0"},"canonical_sha256":"f56ee44bd3730b1015904c5ddba77922b07252113b6d9c76f1bc2919ead632f6","source":{"kind":"arxiv","id":"2407.21787","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.21787","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"arxiv_version","alias_value":"2407.21787v3","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.21787","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"pith_short_12","alias_value":"6VXOIS6TOMFR","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"pith_short_16","alias_value":"6VXOIS6TOMFRAFMQ","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"pith_short_8","alias_value":"6VXOIS6T","created_at":"2026-07-05T09:55:34Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:6VXOIS6TOMFRAFMQJRO5XJ3ZEK","target":"record","payload":{"canonical_record":{"source":{"id":"2407.21787","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-31T17:57:25Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"31f0dd143626a168bc507ded074aa4847baab69be95bca12326b9be6945ff850","abstract_canon_sha256":"2ea7eaa9bcea62249d410bfcd9d73e3b07ae9655cf37510b4beaa4da2daecd5b"},"schema_version":"1.0"},"canonical_sha256":"f56ee44bd3730b1015904c5ddba77922b07252113b6d9c76f1bc2919ead632f6","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:55:34.105390Z","signature_b64":"/6wpNyAjYNOcx14CJDcKx1skGv0zpoSx88CtdFOWqk9A7wsW90vMxo4Hfex5Jyl8bVbuOeroNzP8yV3w4LMGAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f56ee44bd3730b1015904c5ddba77922b07252113b6d9c76f1bc2919ead632f6","last_reissued_at":"2026-07-05T09:55:34.104858Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:55:34.104858Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.21787","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:55:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Pd0ekbwGNSKgNbchCacTEzSPhMKdrJaGY0sK//kffHpqGv+PMllK3NBzgnDOGKbNqyrvkuGfFnyjgCJMeJX8CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T01:29:43.323567Z"},"content_sha256":"66d4a9d0b8485fefc503367812617e476f86e46ba54eb031cf93b5b6d30672e7","schema_version":"1.0","event_id":"sha256:66d4a9d0b8485fefc503367812617e476f86e46ba54eb031cf93b5b6d30672e7"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:6VXOIS6TOMFRAFMQJRO5XJ3ZEK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Large Language Monkeys: Scaling Inference Compute with Repeated Sampling","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Repeatedly sampling from language models scales the fraction of problems solved over four orders of magnitude.","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Azalia Mirhoseini, Bradley Brown, Christopher R\\'e, Jordan Juravsky, Quoc V. Le, Ronald Clark, Ryan Ehrlich","submitted_at":"2024-07-31T17:57:25Z","abstract_excerpt":"Scaling the amount of compute used to train language models has dramatically improved their capabilities. However, when it comes to inference, we often limit models to making only one attempt at a problem. Here, we explore inference compute as another axis for scaling, using the simple technique of repeatedly sampling candidate solutions from a model. Across multiple tasks and models, we observe that coverage -- the fraction of problems that are solved by any generated sample -- scales with the number of samples over four orders of magnitude. Interestingly, the relationship between coverage an"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"coverage -- the fraction of problems that are solved by any generated sample -- scales with the number of samples over four orders of magnitude. Interestingly, the relationship between coverage and the number of samples is often log-linear and can be modelled with an exponentiated power law, suggesting the existence of inference-time scaling laws.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That automatic verification is available to identify correct samples from the collection, and that the observed scaling holds without the model collapsing to repetitive outputs.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"Repeated sampling scales problem coverage log-linearly with sample count, improving SWE-bench Lite performance from 15.9% to 56% using 250 samples.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Repeatedly sampling from language models scales the fraction of problems solved over four orders of magnitude.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"61d2ae16faa648f8c220dc405c07d182e856ce8294c6d51c834340f35e5952f2"},"source":{"id":"2407.21787","kind":"arxiv","version":3},"verdict":{"id":"db2cba80-0aef-4727-8d7d-efcc27f9a1fb","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-12T04:37:53.359354Z","strongest_claim":"coverage -- the fraction of problems that are solved by any generated sample -- scales with the number of samples over four orders of magnitude. Interestingly, the relationship between coverage and the number of samples is often log-linear and can be modelled with an exponentiated power law, suggesting the existence of inference-time scaling laws.","one_line_summary":"Repeated sampling scales problem coverage log-linearly with sample count, improving SWE-bench Lite performance from 15.9% to 56% using 250 samples.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That automatic verification is available to identify correct samples from the collection, and that the observed scaling holds without the model collapsing to repetitive outputs.","pith_extraction_headline":"Repeatedly sampling from language models scales the fraction of problems solved over four orders of magnitude."},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.21787/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":72,"sample":[{"doi":"","year":2024,"title":"URL https://aide.dev/","work_id":"32e0a68a-6223-434c-875a-950bb6ecb673","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2024,"title":"URL https://openai.com/index/hello-gpt-4o/","work_id":"caf3e378-1f89-4117-bf7b-a70a340ba435","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2024,"title":"URL https://llama.meta.com/llama3/","work_id":"fcec34f7-4120-4464-a668-68b13f570ef3","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2024,"title":"URL https://www.anthropic.com/news/claude-3-5-sonnet","work_id":"026d63d0-cf0d-4504-9ef7-99acc9cfd456","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2024,"title":"URL https://www.voyageai.com/","work_id":"c2e9efed-6874-4350-9617-47d981771823","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":72,"snapshot_sha256":"e4a774999c0707453a140b61b49effde78f90b97ed2c4d22ee7f00de85c65b67","internal_anchors":22},"formal_canon":{"evidence_count":2,"snapshot_sha256":"d07e3f5531c7ee94c4347d2ca8b459a238e6cc8125fa30eb659f14374f47e789"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"db2cba80-0aef-4727-8d7d-efcc27f9a1fb"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:55:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"y3P+bmRuFiDMRTjaVKgFS+YVrLKm+zEoh4w3KNdwdwjnmQDLYkXpqR2EbusD4z5mDCa8rMs2DVNbkZ2IHp5LCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T01:29:43.324346Z"},"content_sha256":"cab39a0ae539afcb1b894f6a904f630b183aa5a2906dd94be38f0163830d7eda","schema_version":"1.0","event_id":"sha256:cab39a0ae539afcb1b894f6a904f630b183aa5a2906dd94be38f0163830d7eda"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6VXOIS6TOMFRAFMQJRO5XJ3ZEK/bundle.json","state_url":"https://pith.science/pith/6VXOIS6TOMFRAFMQJRO5XJ3ZEK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6VXOIS6TOMFRAFMQJRO5XJ3ZEK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T01:29:43Z","links":{"resolver":"https://pith.science/pith/6VXOIS6TOMFRAFMQJRO5XJ3ZEK","bundle":"https://pith.science/pith/6VXOIS6TOMFRAFMQJRO5XJ3ZEK/bundle.json","state":"https://pith.science/pith/6VXOIS6TOMFRAFMQJRO5XJ3ZEK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6VXOIS6TOMFRAFMQJRO5XJ3ZEK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:6VXOIS6TOMFRAFMQJRO5XJ3ZEK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2ea7eaa9bcea62249d410bfcd9d73e3b07ae9655cf37510b4beaa4da2daecd5b","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-31T17:57:25Z","title_canon_sha256":"31f0dd143626a168bc507ded074aa4847baab69be95bca12326b9be6945ff850"},"schema_version":"1.0","source":{"id":"2407.21787","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.21787","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"arxiv_version","alias_value":"2407.21787v3","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.21787","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"pith_short_12","alias_value":"6VXOIS6TOMFR","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"pith_short_16","alias_value":"6VXOIS6TOMFRAFMQ","created_at":"2026-07-05T09:55:34Z"},{"alias_kind":"pith_short_8","alias_value":"6VXOIS6T","created_at":"2026-07-05T09:55:34Z"}],"graph_snapshots":[{"event_id":"sha256:cab39a0ae539afcb1b894f6a904f630b183aa5a2906dd94be38f0163830d7eda","target":"graph","created_at":"2026-07-05T09:55:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"coverage -- the fraction of problems that are solved by any generated sample -- scales with the number of samples over four orders of magnitude. Interestingly, the relationship between coverage and the number of samples is often log-linear and can be modelled with an exponentiated power law, suggesting the existence of inference-time scaling laws."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That automatic verification is available to identify correct samples from the collection, and that the observed scaling holds without the model collapsing to repetitive outputs."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"Repeated sampling scales problem coverage log-linearly with sample count, improving SWE-bench Lite performance from 15.9% to 56% using 250 samples."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Repeatedly sampling from language models scales the fraction of problems solved over four orders of magnitude."}],"snapshot_sha256":"61d2ae16faa648f8c220dc405c07d182e856ce8294c6d51c834340f35e5952f2"},"formal_canon":{"evidence_count":2,"snapshot_sha256":"d07e3f5531c7ee94c4347d2ca8b459a238e6cc8125fa30eb659f14374f47e789"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.21787/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Scaling the amount of compute used to train language models has dramatically improved their capabilities. However, when it comes to inference, we often limit models to making only one attempt at a problem. Here, we explore inference compute as another axis for scaling, using the simple technique of repeatedly sampling candidate solutions from a model. Across multiple tasks and models, we observe that coverage -- the fraction of problems that are solved by any generated sample -- scales with the number of samples over four orders of magnitude. Interestingly, the relationship between coverage an","authors_text":"Azalia Mirhoseini, Bradley Brown, Christopher R\\'e, Jordan Juravsky, Quoc V. Le, Ronald Clark, Ryan Ehrlich","cross_cats":["cs.AI"],"headline":"Repeatedly sampling from language models scales the fraction of problems solved over four orders of magnitude.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-31T17:57:25Z","title":"Large Language Monkeys: Scaling Inference Compute with Repeated Sampling"},"references":{"count":72,"internal_anchors":22,"resolved_work":72,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"URL https://aide.dev/","work_id":"32e0a68a-6223-434c-875a-950bb6ecb673","year":2024},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"URL https://openai.com/index/hello-gpt-4o/","work_id":"caf3e378-1f89-4117-bf7b-a70a340ba435","year":2024},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":3,"title":"URL https://llama.meta.com/llama3/","work_id":"fcec34f7-4120-4464-a668-68b13f570ef3","year":2024},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"URL https://www.anthropic.com/news/claude-3-5-sonnet","work_id":"026d63d0-cf0d-4504-9ef7-99acc9cfd456","year":2024},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"URL https://www.voyageai.com/","work_id":"c2e9efed-6874-4350-9617-47d981771823","year":2024}],"snapshot_sha256":"e4a774999c0707453a140b61b49effde78f90b97ed2c4d22ee7f00de85c65b67"},"source":{"id":"2407.21787","kind":"arxiv","version":3},"verdict":{"created_at":"2026-05-12T04:37:53.359354Z","id":"db2cba80-0aef-4727-8d7d-efcc27f9a1fb","model_set":{"reader":"grok-4.3"},"one_line_summary":"Repeated sampling scales problem coverage log-linearly with sample count, improving SWE-bench Lite performance from 15.9% to 56% using 250 samples.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Repeatedly sampling from language models scales the fraction of problems solved over four orders of magnitude.","strongest_claim":"coverage -- the fraction of problems that are solved by any generated sample -- scales with the number of samples over four orders of magnitude. Interestingly, the relationship between coverage and the number of samples is often log-linear and can be modelled with an exponentiated power law, suggesting the existence of inference-time scaling laws.","weakest_assumption":"That automatic verification is available to identify correct samples from the collection, and that the observed scaling holds without the model collapsing to repetitive outputs."}},"verdict_id":"db2cba80-0aef-4727-8d7d-efcc27f9a1fb"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:66d4a9d0b8485fefc503367812617e476f86e46ba54eb031cf93b5b6d30672e7","target":"record","created_at":"2026-07-05T09:55:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2ea7eaa9bcea62249d410bfcd9d73e3b07ae9655cf37510b4beaa4da2daecd5b","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-07-31T17:57:25Z","title_canon_sha256":"31f0dd143626a168bc507ded074aa4847baab69be95bca12326b9be6945ff850"},"schema_version":"1.0","source":{"id":"2407.21787","kind":"arxiv","version":3}},"canonical_sha256":"f56ee44bd3730b1015904c5ddba77922b07252113b6d9c76f1bc2919ead632f6","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f56ee44bd3730b1015904c5ddba77922b07252113b6d9c76f1bc2919ead632f6","first_computed_at":"2026-07-05T09:55:34.104858Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:55:34.104858Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"/6wpNyAjYNOcx14CJDcKx1skGv0zpoSx88CtdFOWqk9A7wsW90vMxo4Hfex5Jyl8bVbuOeroNzP8yV3w4LMGAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:55:34.105390Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.21787","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:66d4a9d0b8485fefc503367812617e476f86e46ba54eb031cf93b5b6d30672e7","sha256:cab39a0ae539afcb1b894f6a904f630b183aa5a2906dd94be38f0163830d7eda"],"state_sha256":"cd7c9143f375acd5bf6f237b0acdde43711688859192beed621d63535bc3620e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ww6p25Achqh/76o7B6e0kJZt1bfKJcD0vOH6G8DQj5gScmfX7v/VUR91GoeCZZEJf4fznUER8K1mW4CQGxyzCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T01:29:43.329247Z","bundle_sha256":"6e8e0b2b7efd9f6ee581cdc9c5ba82948f739cef27757c92bc2da348b88fd629"}}