{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:IXC5F7PDSFYDQVS7IOM4RZQ44W","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fb9af92dfca3e190b14c7d16e3b1baa668cd0a65acac7d79474cfa6ea5f82fdd","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T15:52:39Z","title_canon_sha256":"06f60e21f4c5e28452ac47ba5048b0184059ccc9bd4edb974b7fefc41ceff51e"},"schema_version":"1.0","source":{"id":"2212.07919","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.07919","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"arxiv_version","alias_value":"2212.07919v2","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.07919","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_12","alias_value":"IXC5F7PDSFYD","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_16","alias_value":"IXC5F7PDSFYDQVS7","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_8","alias_value":"IXC5F7PD","created_at":"2026-07-05T06:49:58Z"}],"graph_snapshots":[{"event_id":"sha256:b6225cf6bc74e9d04ba66afe2d5945ca38c4f8c9f0366ace8e1866232efb501c","target":"graph","created_at":"2026-07-05T06:49:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2212.07919/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models show improved downstream task performance when prompted to generate step-by-step reasoning to justify their final answers. These reasoning steps greatly improve model interpretability and verification, but objectively studying their correctness (independent of the final answer) is difficult without reliable methods for automatic evaluation. We simply do not know how often the stated reasoning steps actually support the final end task predictions. In this work, we present ROSCOE, a suite of interpretable, unsupervised automatic scores that improve and extend previous text ","authors_text":"Asli Celikyilmaz, Luke Zettlemoyer, Martin Corredor, Maryam Fazel-Zarandi, Moya Chen, Olga Golovneva, Spencer Poff","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T15:52:39Z","title":"ROSCOE: A Suite of Metrics for Scoring Step-by-Step Reasoning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.07919","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:59cb5dc57eb9a30e6b4b5f6f9c6a444f1fef36eb31be74da68b305f172ec87fe","target":"record","created_at":"2026-07-05T06:49:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fb9af92dfca3e190b14c7d16e3b1baa668cd0a65acac7d79474cfa6ea5f82fdd","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T15:52:39Z","title_canon_sha256":"06f60e21f4c5e28452ac47ba5048b0184059ccc9bd4edb974b7fefc41ceff51e"},"schema_version":"1.0","source":{"id":"2212.07919","kind":"arxiv","version":2}},"canonical_sha256":"45c5d2fde3917038565f4399c8e61ce5a712815543618bf19c58ff55ebc23e76","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"45c5d2fde3917038565f4399c8e61ce5a712815543618bf19c58ff55ebc23e76","first_computed_at":"2026-07-05T06:49:58.595222Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:49:58.595222Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nPC9fQWSvCH9CFBk0abk3knmyZpTkYkK6G7V7TtdZWGC2JxQV6Fh6UJeDEdO7m6OFLAiWdpfuQt6KkBhJfnIAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:49:58.595849Z","signed_message":"canonical_sha256_bytes"},"source_id":"2212.07919","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:59cb5dc57eb9a30e6b4b5f6f9c6a444f1fef36eb31be74da68b305f172ec87fe","sha256:b6225cf6bc74e9d04ba66afe2d5945ca38c4f8c9f0366ace8e1866232efb501c"],"state_sha256":"5ec781b6a8cb49ca752115c9c646d6671cc8fb9c300074efe26e80b08df96ade"}