{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:6MZAKTR6BJYSAZZRFOM4GQEUQ2","short_pith_number":"pith:6MZAKTR6","canonical_record":{"source":{"id":"2607.27518","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-07-29T23:17:08Z","cross_cats_sorted":[],"title_canon_sha256":"c945562c98ac185afe4ae4c9cbbb720060a6e5185d7c4d71f527c0ef182e2681","abstract_canon_sha256":"2a40d86c9b01c461dae47a4da99c0285900e22ab61faba7f40a20afd65e2f60d"},"schema_version":"1.0"},"canonical_sha256":"f332054e3e0a712067312b99c34094869cc39b1698eca4a027d6a879e67eb4bd","source":{"kind":"arxiv","id":"2607.27518","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2607.27518","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"arxiv_version","alias_value":"2607.27518v1","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2607.27518","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"pith_short_12","alias_value":"6MZAKTR6BJYS","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"pith_short_16","alias_value":"6MZAKTR6BJYSAZZR","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"pith_short_8","alias_value":"6MZAKTR6","created_at":"2026-07-31T01:24:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:6MZAKTR6BJYSAZZRFOM4GQEUQ2","target":"record","payload":{"canonical_record":{"source":{"id":"2607.27518","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-07-29T23:17:08Z","cross_cats_sorted":[],"title_canon_sha256":"c945562c98ac185afe4ae4c9cbbb720060a6e5185d7c4d71f527c0ef182e2681","abstract_canon_sha256":"2a40d86c9b01c461dae47a4da99c0285900e22ab61faba7f40a20afd65e2f60d"},"schema_version":"1.0"},"canonical_sha256":"f332054e3e0a712067312b99c34094869cc39b1698eca4a027d6a879e67eb4bd","receipt":{"kind":"pith_receipt","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f332054e3e0a712067312b99c34094869cc39b1698eca4a027d6a879e67eb4bd","last_reissued_at":"2026-07-31T01:24:58.305409Z","signature_status":"unsigned_v0","first_computed_at":"2026-07-31T01:24:58.305409Z"},"source_kind":"arxiv","source_id":"2607.27518","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-31T01:24:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"L71xnW7YWMyLNhDNGUauWaNxSFLV11bf9246NMOS1CcnlkQrMJuLWXEl76SUEFMSAk8GUJK8bDapLbY+ZLB6Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T03:09:17.664543Z"},"content_sha256":"63d773baf57cb224441a87557d50075b5fcfc17ae84e4919911f12012c7e1f11","schema_version":"1.0","event_id":"sha256:63d773baf57cb224441a87557d50075b5fcfc17ae84e4919911f12012c7e1f11"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:6MZAKTR6BJYSAZZRFOM4GQEUQ2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Automated Transcript Analysis for Detecting Flaws in Agentic Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Benjamin Allan-Rahill, Damian S\\'ojka, Harry Coppock, James Mann, Jeff Mohl, Justin Olive, Kaelan Yim, Magda Dubois, Nelson Gardner-Challis","submitted_at":"2026-07-29T23:17:08Z","abstract_excerpt":"Capabilities of frontier models are often assessed using agentic benchmarks. To trust these results, benchmarks must accurately measure what they claim to and be free from invalidating flaws. Previous manual audits of benchmarks such as SWE-Bench-Verified have uncovered several validity issues in transcripts. However, manual review is difficult to scale, and it is unclear whether automated methods can reliably surface flaws that compromise benchmark validity. In this paper, we developed AI scanners to detect four types of validity issues: ground truth access, tool failure, guessing vulnerabili"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2607.27518","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2607.27518/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-31T01:24:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3II0Q1RsAqslCO0dLgiXkaUp/hIinYkgZbphwXFOVz0eEmgvIMvADbreUpiaYmjQXtmCrm6Ifd/9kd9jfTW7Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T03:09:17.664931Z"},"content_sha256":"4162523d24d693ab7657c2bf59638c01ef2bc5b9cf446a9ee55f7032e5fb02b8","schema_version":"1.0","event_id":"sha256:4162523d24d693ab7657c2bf59638c01ef2bc5b9cf446a9ee55f7032e5fb02b8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6MZAKTR6BJYSAZZRFOM4GQEUQ2/bundle.json","state_url":"https://pith.science/pith/6MZAKTR6BJYSAZZRFOM4GQEUQ2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6MZAKTR6BJYSAZZRFOM4GQEUQ2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T03:09:17Z","links":{"resolver":"https://pith.science/pith/6MZAKTR6BJYSAZZRFOM4GQEUQ2","bundle":"https://pith.science/pith/6MZAKTR6BJYSAZZRFOM4GQEUQ2/bundle.json","state":"https://pith.science/pith/6MZAKTR6BJYSAZZRFOM4GQEUQ2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6MZAKTR6BJYSAZZRFOM4GQEUQ2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:6MZAKTR6BJYSAZZRFOM4GQEUQ2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2a40d86c9b01c461dae47a4da99c0285900e22ab61faba7f40a20afd65e2f60d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-07-29T23:17:08Z","title_canon_sha256":"c945562c98ac185afe4ae4c9cbbb720060a6e5185d7c4d71f527c0ef182e2681"},"schema_version":"1.0","source":{"id":"2607.27518","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2607.27518","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"arxiv_version","alias_value":"2607.27518v1","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2607.27518","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"pith_short_12","alias_value":"6MZAKTR6BJYS","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"pith_short_16","alias_value":"6MZAKTR6BJYSAZZR","created_at":"2026-07-31T01:24:58Z"},{"alias_kind":"pith_short_8","alias_value":"6MZAKTR6","created_at":"2026-07-31T01:24:58Z"}],"graph_snapshots":[{"event_id":"sha256:4162523d24d693ab7657c2bf59638c01ef2bc5b9cf446a9ee55f7032e5fb02b8","target":"graph","created_at":"2026-07-31T01:24:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2607.27518/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Capabilities of frontier models are often assessed using agentic benchmarks. To trust these results, benchmarks must accurately measure what they claim to and be free from invalidating flaws. Previous manual audits of benchmarks such as SWE-Bench-Verified have uncovered several validity issues in transcripts. However, manual review is difficult to scale, and it is unclear whether automated methods can reliably surface flaws that compromise benchmark validity. In this paper, we developed AI scanners to detect four types of validity issues: ground truth access, tool failure, guessing vulnerabili","authors_text":"Benjamin Allan-Rahill, Damian S\\'ojka, Harry Coppock, James Mann, Jeff Mohl, Justin Olive, Kaelan Yim, Magda Dubois, Nelson Gardner-Challis","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-07-29T23:17:08Z","title":"Automated Transcript Analysis for Detecting Flaws in Agentic Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2607.27518","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:63d773baf57cb224441a87557d50075b5fcfc17ae84e4919911f12012c7e1f11","target":"record","created_at":"2026-07-31T01:24:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2a40d86c9b01c461dae47a4da99c0285900e22ab61faba7f40a20afd65e2f60d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2026-07-29T23:17:08Z","title_canon_sha256":"c945562c98ac185afe4ae4c9cbbb720060a6e5185d7c4d71f527c0ef182e2681"},"schema_version":"1.0","source":{"id":"2607.27518","kind":"arxiv","version":1}},"canonical_sha256":"f332054e3e0a712067312b99c34094869cc39b1698eca4a027d6a879e67eb4bd","receipt":{"builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f332054e3e0a712067312b99c34094869cc39b1698eca4a027d6a879e67eb4bd","first_computed_at":"2026-07-31T01:24:58.305409Z","kind":"pith_receipt","last_reissued_at":"2026-07-31T01:24:58.305409Z","receipt_version":"0.3","signature_status":"unsigned_v0"},"source_id":"2607.27518","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:63d773baf57cb224441a87557d50075b5fcfc17ae84e4919911f12012c7e1f11","sha256:4162523d24d693ab7657c2bf59638c01ef2bc5b9cf446a9ee55f7032e5fb02b8"],"state_sha256":"1578807a50602438ba6fa6057394182f49704a6c3b6e6ba7fbfdb78979f37448"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BmbAL0xkjCPIqEFDtRoGxc8jJnZG1Y+4fGJvDGnMIVDeGfZQLJ1Ea5FhMY5bAq7ohbDmEilFBO8i3RBSdGsCAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T03:09:17.671957Z","bundle_sha256":"8c776b74e2d5ed34331ffaaf8b508ae1a26c1564bbe8b38ca855e92120d0b89c"}}