{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:BEHL4VYMCGL7EG5Q3QFZZ3WY5K","short_pith_number":"pith:BEHL4VYM","canonical_record":{"source":{"id":"2412.18544","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-12-24T16:51:35Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"947e31dfe89de9d0e1020d8642137495d860c5c5d5e55aefd0dd1519db2432ec","abstract_canon_sha256":"46de5aaae8aa8aede29143fbd9c290fdc5b069bb592a76471122fc71f8a7fcf7"},"schema_version":"1.0"},"canonical_sha256":"090ebe570c1197f21bb0dc0b9ceed8ea92c314b1c7bbfbb8b6e3d7b72f5bfefe","source":{"kind":"arxiv","id":"2412.18544","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.18544","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"arxiv_version","alias_value":"2412.18544v2","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.18544","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"pith_short_12","alias_value":"BEHL4VYMCGL7","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"pith_short_16","alias_value":"BEHL4VYMCGL7EG5Q","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"pith_short_8","alias_value":"BEHL4VYM","created_at":"2026-07-05T09:59:21Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:BEHL4VYMCGL7EG5Q3QFZZ3WY5K","target":"record","payload":{"canonical_record":{"source":{"id":"2412.18544","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-12-24T16:51:35Z","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"title_canon_sha256":"947e31dfe89de9d0e1020d8642137495d860c5c5d5e55aefd0dd1519db2432ec","abstract_canon_sha256":"46de5aaae8aa8aede29143fbd9c290fdc5b069bb592a76471122fc71f8a7fcf7"},"schema_version":"1.0"},"canonical_sha256":"090ebe570c1197f21bb0dc0b9ceed8ea92c314b1c7bbfbb8b6e3d7b72f5bfefe","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:59:21.779709Z","signature_b64":"3dif0jlcYenbxLBzm+HbT3fhS3dAvUfhmi4+MBoQmRq7Wa2DegIHU33w8nH15aWN51lqE12RwB/KKlEN/uNlCQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"090ebe570c1197f21bb0dc0b9ceed8ea92c314b1c7bbfbb8b6e3d7b72f5bfefe","last_reissued_at":"2026-07-05T09:59:21.779158Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:59:21.779158Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2412.18544","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:59:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"t0iUbfL5TO6MK3K6LtJwKy/HKCJGYfTKgr2TeQOrH/UmQCuIyNTwCBEhjEZwjl3aG/qJ8jHtLcY7YGaRd3c7Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T04:02:05.141724Z"},"content_sha256":"e506b74c65ea08e33843136e6e4782f62c14bd5bf91682a168b5d3d999bd8ddd","schema_version":"1.0","event_id":"sha256:e506b74c65ea08e33843136e6e4782f62c14bd5bf91682a168b5d3d999bd8ddd"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:BEHL4VYMCGL7EG5Q3QFZZ3WY5K","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Consistency Checks for Language Model Forecasters","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.CL","stat.ML"],"primary_cat":"cs.LG","authors_text":"Abhimanyu Pallavi Sudhir, Adam Shen, Alejandro Alvarez, Daniel Paleka, Evan Wang, Florian Tram\\`er, Vineeth Bhat","submitted_at":"2024-12-24T16:51:35Z","abstract_excerpt":"Forecasting is a task that is difficult to evaluate: the ground truth can only be known in the future. Recent work showing LLM forecasters rapidly approaching human-level performance begs the question: how can we benchmark and evaluate these forecasters instantaneously? Following the consistency check framework, we measure the performance of forecasters in terms of the consistency of their predictions on different logically-related questions. We propose a new, general consistency metric based on arbitrage: for example, if a forecasting AI illogically predicts that both the Democratic and Repub"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.18544","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.18544/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:59:21Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Hfv2v0Odbvf4QL7nuOz1+wdHc1P9rEiCKkbjDbcapEY22Qfl/d/eYaeit1aqvjaG04e5p6Z1Xf/XRi3ZxR/mDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T04:02:05.142249Z"},"content_sha256":"7fca15af724a6a0770801c3cdf8cb4abf8d5646c6dfe6938cdf8abcb222bb034","schema_version":"1.0","event_id":"sha256:7fca15af724a6a0770801c3cdf8cb4abf8d5646c6dfe6938cdf8abcb222bb034"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BEHL4VYMCGL7EG5Q3QFZZ3WY5K/bundle.json","state_url":"https://pith.science/pith/BEHL4VYMCGL7EG5Q3QFZZ3WY5K/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BEHL4VYMCGL7EG5Q3QFZZ3WY5K/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T04:02:05Z","links":{"resolver":"https://pith.science/pith/BEHL4VYMCGL7EG5Q3QFZZ3WY5K","bundle":"https://pith.science/pith/BEHL4VYMCGL7EG5Q3QFZZ3WY5K/bundle.json","state":"https://pith.science/pith/BEHL4VYMCGL7EG5Q3QFZZ3WY5K/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BEHL4VYMCGL7EG5Q3QFZZ3WY5K/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:BEHL4VYMCGL7EG5Q3QFZZ3WY5K","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"46de5aaae8aa8aede29143fbd9c290fdc5b069bb592a76471122fc71f8a7fcf7","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-12-24T16:51:35Z","title_canon_sha256":"947e31dfe89de9d0e1020d8642137495d860c5c5d5e55aefd0dd1519db2432ec"},"schema_version":"1.0","source":{"id":"2412.18544","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.18544","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"arxiv_version","alias_value":"2412.18544v2","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.18544","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"pith_short_12","alias_value":"BEHL4VYMCGL7","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"pith_short_16","alias_value":"BEHL4VYMCGL7EG5Q","created_at":"2026-07-05T09:59:21Z"},{"alias_kind":"pith_short_8","alias_value":"BEHL4VYM","created_at":"2026-07-05T09:59:21Z"}],"graph_snapshots":[{"event_id":"sha256:7fca15af724a6a0770801c3cdf8cb4abf8d5646c6dfe6938cdf8abcb222bb034","target":"graph","created_at":"2026-07-05T09:59:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.18544/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Forecasting is a task that is difficult to evaluate: the ground truth can only be known in the future. Recent work showing LLM forecasters rapidly approaching human-level performance begs the question: how can we benchmark and evaluate these forecasters instantaneously? Following the consistency check framework, we measure the performance of forecasters in terms of the consistency of their predictions on different logically-related questions. We propose a new, general consistency metric based on arbitrage: for example, if a forecasting AI illogically predicts that both the Democratic and Repub","authors_text":"Abhimanyu Pallavi Sudhir, Adam Shen, Alejandro Alvarez, Daniel Paleka, Evan Wang, Florian Tram\\`er, Vineeth Bhat","cross_cats":["cs.AI","cs.CL","stat.ML"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-12-24T16:51:35Z","title":"Consistency Checks for Language Model Forecasters"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.18544","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e506b74c65ea08e33843136e6e4782f62c14bd5bf91682a168b5d3d999bd8ddd","target":"record","created_at":"2026-07-05T09:59:21Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"46de5aaae8aa8aede29143fbd9c290fdc5b069bb592a76471122fc71f8a7fcf7","cross_cats_sorted":["cs.AI","cs.CL","stat.ML"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-12-24T16:51:35Z","title_canon_sha256":"947e31dfe89de9d0e1020d8642137495d860c5c5d5e55aefd0dd1519db2432ec"},"schema_version":"1.0","source":{"id":"2412.18544","kind":"arxiv","version":2}},"canonical_sha256":"090ebe570c1197f21bb0dc0b9ceed8ea92c314b1c7bbfbb8b6e3d7b72f5bfefe","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"090ebe570c1197f21bb0dc0b9ceed8ea92c314b1c7bbfbb8b6e3d7b72f5bfefe","first_computed_at":"2026-07-05T09:59:21.779158Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:59:21.779158Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3dif0jlcYenbxLBzm+HbT3fhS3dAvUfhmi4+MBoQmRq7Wa2DegIHU33w8nH15aWN51lqE12RwB/KKlEN/uNlCQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:59:21.779709Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.18544","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e506b74c65ea08e33843136e6e4782f62c14bd5bf91682a168b5d3d999bd8ddd","sha256:7fca15af724a6a0770801c3cdf8cb4abf8d5646c6dfe6938cdf8abcb222bb034"],"state_sha256":"0bc41b3dea58a31187f206a89749f44bf5d52eb3a0cec9cf8c0056d74f674c91"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tkF+nZsx0VUJj4MXRI0ZdwVIPk1D3kQa7iidIsOGDJYwNeq0V0MYrzjzOBHwf8fcddwyWQSAt9n0WOCOV1VcCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T04:02:05.147549Z","bundle_sha256":"4caff7485a8dc8d31453555eab98440af83b87b7a39bf413ba300946c207a766"}}