{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:VX6U5PI4DEGZC63ZJWGUJ4IL22","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5e7c540d8d7dbdfe55919795a1fbeac2f5c4eee9d237fe3253954c8857a36016","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-12-29T07:27:23Z","title_canon_sha256":"6b8080ba7f9a8a12659545f9a272c67ac03ebe92293011994901219c7e407fc4"},"schema_version":"1.0","source":{"id":"1912.12598","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1912.12598","created_at":"2026-07-05T00:29:01Z"},{"alias_kind":"arxiv_version","alias_value":"1912.12598v1","created_at":"2026-07-05T00:29:01Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1912.12598","created_at":"2026-07-05T00:29:01Z"},{"alias_kind":"pith_short_12","alias_value":"VX6U5PI4DEGZ","created_at":"2026-07-05T00:29:01Z"},{"alias_kind":"pith_short_16","alias_value":"VX6U5PI4DEGZC63Z","created_at":"2026-07-05T00:29:01Z"},{"alias_kind":"pith_short_8","alias_value":"VX6U5PI4","created_at":"2026-07-05T00:29:01Z"}],"graph_snapshots":[{"event_id":"sha256:4b2edb2442bb88cf73d344f16d9c866549369ef6d3593cd4cbb0cc4253309b37","target":"graph","created_at":"2026-07-05T00:29:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/1912.12598/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reading comprehension is one of the crucial tasks for furthering research in natural language understanding. A lot of diverse reading comprehension datasets have recently been introduced to study various phenomena in natural language, ranging from simple paraphrase matching and entity typing to entity tracking and understanding the implications of the context. Given the availability of many such datasets, comprehensive and reliable evaluation is tedious and time-consuming for researchers working on this problem. We present an evaluation server, ORB, that reports performance on seven diverse re","authors_text":"Alon Talmor, Ananth Gottumukkala, Dheeru Dua, Matt Gardner, Sameer Singh","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-12-29T07:27:23Z","title":"ORB: An Open Reading Benchmark for Comprehensive Evaluation of Machine Reading Comprehension"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1912.12598","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f6cdde031e052cd6d64d6bafe8285ad5417d0eb7ddac3e7d3abde275796059e1","target":"record","created_at":"2026-07-05T00:29:01Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5e7c540d8d7dbdfe55919795a1fbeac2f5c4eee9d237fe3253954c8857a36016","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2019-12-29T07:27:23Z","title_canon_sha256":"6b8080ba7f9a8a12659545f9a272c67ac03ebe92293011994901219c7e407fc4"},"schema_version":"1.0","source":{"id":"1912.12598","kind":"arxiv","version":1}},"canonical_sha256":"adfd4ebd1c190d917b794d8d44f10bd6b2186edff719b3a165a781514ae389d3","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"adfd4ebd1c190d917b794d8d44f10bd6b2186edff719b3a165a781514ae389d3","first_computed_at":"2026-07-05T00:29:01.469684Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T00:29:01.469684Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"s1SsQlWho1t81AoIoTisFX/qI5ZbE9sj9wUS4P7U46R5RyQN2BqwfWOGiY0UfEaN5s3l8p+t0BXPrZeF7C/NDw==","signature_status":"signed_v1","signed_at":"2026-07-05T00:29:01.470154Z","signed_message":"canonical_sha256_bytes"},"source_id":"1912.12598","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f6cdde031e052cd6d64d6bafe8285ad5417d0eb7ddac3e7d3abde275796059e1","sha256:4b2edb2442bb88cf73d344f16d9c866549369ef6d3593cd4cbb0cc4253309b37"],"state_sha256":"75ab1eaf960ec903c0a501496383cd00385122c0dff8c82c1580e3c19f9a859e"}