{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:7ZY6QH5ZPP7KAVL7XOKNK6KDHW","short_pith_number":"pith:7ZY6QH5Z","canonical_record":{"source":{"id":"2507.02825","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-07-03T17:35:31Z","cross_cats_sorted":[],"title_canon_sha256":"c8b1a22ca24f10df9de7351a2b661a61d126e1b681e62dc12d89f3fd59590a3d","abstract_canon_sha256":"de49248e866a9846d5aaded435610b6ba03831b7d62b6e0eb55a2f8fb9d882c0"},"schema_version":"1.0"},"canonical_sha256":"fe71e81fb97bfea0557fbb94d579433d90a4b04dbe556e43d557f8dbc817143b","source":{"kind":"arxiv","id":"2507.02825","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.02825","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"arxiv_version","alias_value":"2507.02825v5","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.02825","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"pith_short_12","alias_value":"7ZY6QH5ZPP7K","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"pith_short_16","alias_value":"7ZY6QH5ZPP7KAVL7","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"pith_short_8","alias_value":"7ZY6QH5Z","created_at":"2026-07-05T11:49:56Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:7ZY6QH5ZPP7KAVL7XOKNK6KDHW","target":"record","payload":{"canonical_record":{"source":{"id":"2507.02825","kind":"arxiv","version":5},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-07-03T17:35:31Z","cross_cats_sorted":[],"title_canon_sha256":"c8b1a22ca24f10df9de7351a2b661a61d126e1b681e62dc12d89f3fd59590a3d","abstract_canon_sha256":"de49248e866a9846d5aaded435610b6ba03831b7d62b6e0eb55a2f8fb9d882c0"},"schema_version":"1.0"},"canonical_sha256":"fe71e81fb97bfea0557fbb94d579433d90a4b04dbe556e43d557f8dbc817143b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:49:56.309209Z","signature_b64":"slIUAE1IA3vlTmWsRTuOpCljDfcTMGnr9P/y6uFeMQdeJo5x/sXgiq1rOOR9yzaVifR0KXFhqTfaRpXqevuZDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fe71e81fb97bfea0557fbb94d579433d90a4b04dbe556e43d557f8dbc817143b","last_reissued_at":"2026-07-05T11:49:56.308409Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:49:56.308409Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2507.02825","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:49:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"A+4lZjL+zl7TnOjCzZEMVbQaAB7LE56E3jhLZVuFZh+rTHPdphbDiKRYcSqtJ5NQmUc/FSSBUwd8k1m91ZRHCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-17T07:38:38.886714Z"},"content_sha256":"bdaf53b47ae8caa901b2875e668467aa80b9a5a9e425d5efd095727d55f2f25e","schema_version":"1.0","event_id":"sha256:bdaf53b47ae8caa901b2875e668467aa80b9a5a9e425d5efd095727d55f2f25e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:7ZY6QH5ZPP7KAVL7XOKNK6KDHW","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Establishing Best Practices for Building Rigorous Agentic Benchmarks","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"Andy Zhang, Antony Kellermann, Cozmin Ududec, Daniel Kang, Fazl Barez, Harry Coppock, Ion Stoica, Jacob Merizian, Jacob Steinhardt, Jasjeet Sekhon, Jwala Dhamala, Kevin Meng, Mario Giulianelli, Matei Zaharia, Percy Liang, Rahul Gupta, Rebecca Weiss, Sarah Schwettmann, Sasha Cui, Sayash Kapoor, Shayne Longpre, Shu Liu, Tengjun Jin, Yada Pruksachatkun, Yuxuan Zhu","submitted_at":"2025-07-03T17:35:31Z","abstract_excerpt":"Benchmarks are essential for quantitatively tracking progress in AI. As AI agents become increasingly capable, researchers and practitioners have introduced agentic benchmarks to evaluate agents on complex, real-world tasks. These benchmarks typically measure agent capabilities by evaluating task outcomes via specific reward designs. However, we show that many agentic benchmarks have issues in task setup or reward design. For example, SWE-bench Verified uses insufficient test cases, while TAU-bench counts empty responses as successful. Such issues can lead to under- or overestimation of agents"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.02825","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.02825/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:49:56Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jllVmDfiPNpDd7z2+/a9aAD+//iqbHFMtSWXGSkE57l4YXMeyWhqKxlWKll3J0V6AvpkUHC0IBYiYBXuScv5DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-17T07:38:38.887672Z"},"content_sha256":"7a161be153a33eebdb1d5497c1f813e7135944929f7cadf95ffa0f8a904444d4","schema_version":"1.0","event_id":"sha256:7a161be153a33eebdb1d5497c1f813e7135944929f7cadf95ffa0f8a904444d4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/7ZY6QH5ZPP7KAVL7XOKNK6KDHW/bundle.json","state_url":"https://pith.science/pith/7ZY6QH5ZPP7KAVL7XOKNK6KDHW/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/7ZY6QH5ZPP7KAVL7XOKNK6KDHW/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-17T07:38:38Z","links":{"resolver":"https://pith.science/pith/7ZY6QH5ZPP7KAVL7XOKNK6KDHW","bundle":"https://pith.science/pith/7ZY6QH5ZPP7KAVL7XOKNK6KDHW/bundle.json","state":"https://pith.science/pith/7ZY6QH5ZPP7KAVL7XOKNK6KDHW/state.json","well_known_bundle":"https://pith.science/.well-known/pith/7ZY6QH5ZPP7KAVL7XOKNK6KDHW/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:7ZY6QH5ZPP7KAVL7XOKNK6KDHW","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"de49248e866a9846d5aaded435610b6ba03831b7d62b6e0eb55a2f8fb9d882c0","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-07-03T17:35:31Z","title_canon_sha256":"c8b1a22ca24f10df9de7351a2b661a61d126e1b681e62dc12d89f3fd59590a3d"},"schema_version":"1.0","source":{"id":"2507.02825","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2507.02825","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"arxiv_version","alias_value":"2507.02825v5","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.02825","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"pith_short_12","alias_value":"7ZY6QH5ZPP7K","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"pith_short_16","alias_value":"7ZY6QH5ZPP7KAVL7","created_at":"2026-07-05T11:49:56Z"},{"alias_kind":"pith_short_8","alias_value":"7ZY6QH5Z","created_at":"2026-07-05T11:49:56Z"}],"graph_snapshots":[{"event_id":"sha256:7a161be153a33eebdb1d5497c1f813e7135944929f7cadf95ffa0f8a904444d4","target":"graph","created_at":"2026-07-05T11:49:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2507.02825/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Benchmarks are essential for quantitatively tracking progress in AI. As AI agents become increasingly capable, researchers and practitioners have introduced agentic benchmarks to evaluate agents on complex, real-world tasks. These benchmarks typically measure agent capabilities by evaluating task outcomes via specific reward designs. However, we show that many agentic benchmarks have issues in task setup or reward design. For example, SWE-bench Verified uses insufficient test cases, while TAU-bench counts empty responses as successful. Such issues can lead to under- or overestimation of agents","authors_text":"Andy Zhang, Antony Kellermann, Cozmin Ududec, Daniel Kang, Fazl Barez, Harry Coppock, Ion Stoica, Jacob Merizian, Jacob Steinhardt, Jasjeet Sekhon, Jwala Dhamala, Kevin Meng, Mario Giulianelli, Matei Zaharia, Percy Liang, Rahul Gupta, Rebecca Weiss, Sarah Schwettmann, Sasha Cui, Sayash Kapoor, Shayne Longpre, Shu Liu, Tengjun Jin, Yada Pruksachatkun, Yuxuan Zhu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-07-03T17:35:31Z","title":"Establishing Best Practices for Building Rigorous Agentic Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.02825","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bdaf53b47ae8caa901b2875e668467aa80b9a5a9e425d5efd095727d55f2f25e","target":"record","created_at":"2026-07-05T11:49:56Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"de49248e866a9846d5aaded435610b6ba03831b7d62b6e0eb55a2f8fb9d882c0","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2025-07-03T17:35:31Z","title_canon_sha256":"c8b1a22ca24f10df9de7351a2b661a61d126e1b681e62dc12d89f3fd59590a3d"},"schema_version":"1.0","source":{"id":"2507.02825","kind":"arxiv","version":5}},"canonical_sha256":"fe71e81fb97bfea0557fbb94d579433d90a4b04dbe556e43d557f8dbc817143b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"fe71e81fb97bfea0557fbb94d579433d90a4b04dbe556e43d557f8dbc817143b","first_computed_at":"2026-07-05T11:49:56.308409Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:49:56.308409Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"slIUAE1IA3vlTmWsRTuOpCljDfcTMGnr9P/y6uFeMQdeJo5x/sXgiq1rOOR9yzaVifR0KXFhqTfaRpXqevuZDg==","signature_status":"signed_v1","signed_at":"2026-07-05T11:49:56.309209Z","signed_message":"canonical_sha256_bytes"},"source_id":"2507.02825","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bdaf53b47ae8caa901b2875e668467aa80b9a5a9e425d5efd095727d55f2f25e","sha256:7a161be153a33eebdb1d5497c1f813e7135944929f7cadf95ffa0f8a904444d4"],"state_sha256":"dcec02548815d71760859c64dd536e250fdf0b54e9ae8ac5fbf6fa4f3477306c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aERPzspHfiXCH2cj7i5ntyei1OhAJn6JfdxmCUPEefmtModusdtPfz9U8jganohQTpkGEivAdxOriNxoCVqGAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-17T07:38:38.894652Z","bundle_sha256":"fe5efd9e7d906021fbc6ea4ca29fba184c1623a28ebf2bd04ec44c6b4894d0b6"}}