{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:6ENAHGX6CSFO5BWDZ4EGUU7OFT","short_pith_number":"pith:6ENAHGX6","canonical_record":{"source":{"id":"2505.17043","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-13T13:04:04Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"feeee1960f051bdd08158f03319417aaf92d0fc47b37381af854420aa801c17c","abstract_canon_sha256":"c62aa24cbcbe02c2658f6851b9a1659868e01f6e4357c4cfda16a0eae092d5d0"},"schema_version":"1.0"},"canonical_sha256":"f11a039afe148aee86c3cf086a53ee2cc905e2f5123f5140236aa244efee9fbd","source":{"kind":"arxiv","id":"2505.17043","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.17043","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"arxiv_version","alias_value":"2505.17043v1","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.17043","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"pith_short_12","alias_value":"6ENAHGX6CSFO","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"pith_short_16","alias_value":"6ENAHGX6CSFO5BWD","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"pith_short_8","alias_value":"6ENAHGX6","created_at":"2026-07-05T11:07:46Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:6ENAHGX6CSFO5BWDZ4EGUU7OFT","target":"record","payload":{"canonical_record":{"source":{"id":"2505.17043","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-13T13:04:04Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"feeee1960f051bdd08158f03319417aaf92d0fc47b37381af854420aa801c17c","abstract_canon_sha256":"c62aa24cbcbe02c2658f6851b9a1659868e01f6e4357c4cfda16a0eae092d5d0"},"schema_version":"1.0"},"canonical_sha256":"f11a039afe148aee86c3cf086a53ee2cc905e2f5123f5140236aa244efee9fbd","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:07:46.057753Z","signature_b64":"zAcfudLaKGgpKVoRg/vlysS6oJsk5VmJ5mUXHl5YsVsHruaT2MN7pSF3jUaht4kggeOSmPiaN9DWnCyCOoH+DQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f11a039afe148aee86c3cf086a53ee2cc905e2f5123f5140236aa244efee9fbd","last_reissued_at":"2026-07-05T11:07:46.057218Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:07:46.057218Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.17043","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:07:46Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"cEzw3XMdpCJoMG91tRSHyjvFH8+h8vHUE9a9R/8qOmzhcynCind6IuTenkrEAh//a6WBkp38Wga0hBZ6QOmGDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T23:02:18.093352Z"},"content_sha256":"f1df99cbf1cef03fee11ce79beeb5b9854b44b27e6a38c0f8c056a0e092fbe84","schema_version":"1.0","event_id":"sha256:f1df99cbf1cef03fee11ce79beeb5b9854b44b27e6a38c0f8c056a0e092fbe84"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:6ENAHGX6CSFO5BWDZ4EGUU7OFT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"QRA++: Quantified Reproducibility Assessment for Common Types of Results in Natural Language Processing","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Anya Belz","submitted_at":"2025-05-13T13:04:04Z","abstract_excerpt":"Reproduction studies reported in NLP provide individual data points which in combination indicate worryingly low levels of reproducibility in the field. Because each reproduction study reports quantitative conclusions based on its own, often not explicitly stated, criteria for reproduction success/failure, the conclusions drawn are hard to interpret, compare, and learn from. In this paper, we present QRA++, a quantitative approach to reproducibility assessment that (i) produces continuous-valued degree of reproducibility assessments at three levels of granularity; (ii) utilises reproducibility"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.17043","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.17043/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:07:46Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tbIo447O6tUTNWNA96FJBJr2TDri/tmrz22KtXlzeirjLOPLPcHbFGX447R69sq2N7RI1ZNdUSfjZ7DgiUaHDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T23:02:18.094291Z"},"content_sha256":"32a783b1cb0d1b429a05758baca3552351b6d256282d76e5084e98e68fa2dbf2","schema_version":"1.0","event_id":"sha256:32a783b1cb0d1b429a05758baca3552351b6d256282d76e5084e98e68fa2dbf2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/6ENAHGX6CSFO5BWDZ4EGUU7OFT/bundle.json","state_url":"https://pith.science/pith/6ENAHGX6CSFO5BWDZ4EGUU7OFT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/6ENAHGX6CSFO5BWDZ4EGUU7OFT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T23:02:18Z","links":{"resolver":"https://pith.science/pith/6ENAHGX6CSFO5BWDZ4EGUU7OFT","bundle":"https://pith.science/pith/6ENAHGX6CSFO5BWDZ4EGUU7OFT/bundle.json","state":"https://pith.science/pith/6ENAHGX6CSFO5BWDZ4EGUU7OFT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/6ENAHGX6CSFO5BWDZ4EGUU7OFT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:6ENAHGX6CSFO5BWDZ4EGUU7OFT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c62aa24cbcbe02c2658f6851b9a1659868e01f6e4357c4cfda16a0eae092d5d0","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-13T13:04:04Z","title_canon_sha256":"feeee1960f051bdd08158f03319417aaf92d0fc47b37381af854420aa801c17c"},"schema_version":"1.0","source":{"id":"2505.17043","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.17043","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"arxiv_version","alias_value":"2505.17043v1","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.17043","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"pith_short_12","alias_value":"6ENAHGX6CSFO","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"pith_short_16","alias_value":"6ENAHGX6CSFO5BWD","created_at":"2026-07-05T11:07:46Z"},{"alias_kind":"pith_short_8","alias_value":"6ENAHGX6","created_at":"2026-07-05T11:07:46Z"}],"graph_snapshots":[{"event_id":"sha256:32a783b1cb0d1b429a05758baca3552351b6d256282d76e5084e98e68fa2dbf2","target":"graph","created_at":"2026-07-05T11:07:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.17043/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reproduction studies reported in NLP provide individual data points which in combination indicate worryingly low levels of reproducibility in the field. Because each reproduction study reports quantitative conclusions based on its own, often not explicitly stated, criteria for reproduction success/failure, the conclusions drawn are hard to interpret, compare, and learn from. In this paper, we present QRA++, a quantitative approach to reproducibility assessment that (i) produces continuous-valued degree of reproducibility assessments at three levels of granularity; (ii) utilises reproducibility","authors_text":"Anya Belz","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-13T13:04:04Z","title":"QRA++: Quantified Reproducibility Assessment for Common Types of Results in Natural Language Processing"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.17043","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f1df99cbf1cef03fee11ce79beeb5b9854b44b27e6a38c0f8c056a0e092fbe84","target":"record","created_at":"2026-07-05T11:07:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c62aa24cbcbe02c2658f6851b9a1659868e01f6e4357c4cfda16a0eae092d5d0","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-13T13:04:04Z","title_canon_sha256":"feeee1960f051bdd08158f03319417aaf92d0fc47b37381af854420aa801c17c"},"schema_version":"1.0","source":{"id":"2505.17043","kind":"arxiv","version":1}},"canonical_sha256":"f11a039afe148aee86c3cf086a53ee2cc905e2f5123f5140236aa244efee9fbd","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f11a039afe148aee86c3cf086a53ee2cc905e2f5123f5140236aa244efee9fbd","first_computed_at":"2026-07-05T11:07:46.057218Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:07:46.057218Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zAcfudLaKGgpKVoRg/vlysS6oJsk5VmJ5mUXHl5YsVsHruaT2MN7pSF3jUaht4kggeOSmPiaN9DWnCyCOoH+DQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:07:46.057753Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.17043","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f1df99cbf1cef03fee11ce79beeb5b9854b44b27e6a38c0f8c056a0e092fbe84","sha256:32a783b1cb0d1b429a05758baca3552351b6d256282d76e5084e98e68fa2dbf2"],"state_sha256":"b498517329593476bff01c565068f83c31d48809e33da0bcd2aec712992f9839"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2hry7Ls6fZAYeTsBSg/dwM9X+TEj5NnuJnAio9Ky2Tado6puTrFV1lfr6xKHKg0L8faMadwrexqojk0mfe2XDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T23:02:18.103271Z","bundle_sha256":"b48f2db8f866fcc490a60bd240fef7169331d541bbe2e0de87129343aa0d1a26"}}