{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:IXC5F7PDSFYDQVS7IOM4RZQ44W","short_pith_number":"pith:IXC5F7PD","canonical_record":{"source":{"id":"2212.07919","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T15:52:39Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"06f60e21f4c5e28452ac47ba5048b0184059ccc9bd4edb974b7fefc41ceff51e","abstract_canon_sha256":"fb9af92dfca3e190b14c7d16e3b1baa668cd0a65acac7d79474cfa6ea5f82fdd"},"schema_version":"1.0"},"canonical_sha256":"45c5d2fde3917038565f4399c8e61ce5a712815543618bf19c58ff55ebc23e76","source":{"kind":"arxiv","id":"2212.07919","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.07919","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"arxiv_version","alias_value":"2212.07919v2","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.07919","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_12","alias_value":"IXC5F7PDSFYD","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_16","alias_value":"IXC5F7PDSFYDQVS7","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_8","alias_value":"IXC5F7PD","created_at":"2026-07-05T06:49:58Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:IXC5F7PDSFYDQVS7IOM4RZQ44W","target":"record","payload":{"canonical_record":{"source":{"id":"2212.07919","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T15:52:39Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"06f60e21f4c5e28452ac47ba5048b0184059ccc9bd4edb974b7fefc41ceff51e","abstract_canon_sha256":"fb9af92dfca3e190b14c7d16e3b1baa668cd0a65acac7d79474cfa6ea5f82fdd"},"schema_version":"1.0"},"canonical_sha256":"45c5d2fde3917038565f4399c8e61ce5a712815543618bf19c58ff55ebc23e76","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:49:58.595849Z","signature_b64":"nPC9fQWSvCH9CFBk0abk3knmyZpTkYkK6G7V7TtdZWGC2JxQV6Fh6UJeDEdO7m6OFLAiWdpfuQt6KkBhJfnIAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"45c5d2fde3917038565f4399c8e61ce5a712815543618bf19c58ff55ebc23e76","last_reissued_at":"2026-07-05T06:49:58.595222Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:49:58.595222Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2212.07919","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:49:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QO+1T/wOFQ3ep1iaoQz+PNfWN6QkISl1/FRpBi3yqLKVNfQGve027w90xUFl7kQPN4hwGCsc/AmIlL5lFVHnAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T17:34:04.324976Z"},"content_sha256":"59cb5dc57eb9a30e6b4b5f6f9c6a444f1fef36eb31be74da68b305f172ec87fe","schema_version":"1.0","event_id":"sha256:59cb5dc57eb9a30e6b4b5f6f9c6a444f1fef36eb31be74da68b305f172ec87fe"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:IXC5F7PDSFYDQVS7IOM4RZQ44W","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ROSCOE: A Suite of Metrics for Scoring Step-by-Step Reasoning","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Asli Celikyilmaz, Luke Zettlemoyer, Martin Corredor, Maryam Fazel-Zarandi, Moya Chen, Olga Golovneva, Spencer Poff","submitted_at":"2022-12-15T15:52:39Z","abstract_excerpt":"Large language models show improved downstream task performance when prompted to generate step-by-step reasoning to justify their final answers. These reasoning steps greatly improve model interpretability and verification, but objectively studying their correctness (independent of the final answer) is difficult without reliable methods for automatic evaluation. We simply do not know how often the stated reasoning steps actually support the final end task predictions. In this work, we present ROSCOE, a suite of interpretable, unsupervised automatic scores that improve and extend previous text "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.07919","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2212.07919/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:49:58Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JywH/tEJq3NjSHVUzGgM1Pwh6xdVA4uJaHyk/JABR183CNXH99XJustCEv600fn+63kbUWfPk2zqyviDvCaQDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T17:34:04.325468Z"},"content_sha256":"b6225cf6bc74e9d04ba66afe2d5945ca38c4f8c9f0366ace8e1866232efb501c","schema_version":"1.0","event_id":"sha256:b6225cf6bc74e9d04ba66afe2d5945ca38c4f8c9f0366ace8e1866232efb501c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IXC5F7PDSFYDQVS7IOM4RZQ44W/bundle.json","state_url":"https://pith.science/pith/IXC5F7PDSFYDQVS7IOM4RZQ44W/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IXC5F7PDSFYDQVS7IOM4RZQ44W/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T17:34:04Z","links":{"resolver":"https://pith.science/pith/IXC5F7PDSFYDQVS7IOM4RZQ44W","bundle":"https://pith.science/pith/IXC5F7PDSFYDQVS7IOM4RZQ44W/bundle.json","state":"https://pith.science/pith/IXC5F7PDSFYDQVS7IOM4RZQ44W/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IXC5F7PDSFYDQVS7IOM4RZQ44W/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:IXC5F7PDSFYDQVS7IOM4RZQ44W","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fb9af92dfca3e190b14c7d16e3b1baa668cd0a65acac7d79474cfa6ea5f82fdd","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T15:52:39Z","title_canon_sha256":"06f60e21f4c5e28452ac47ba5048b0184059ccc9bd4edb974b7fefc41ceff51e"},"schema_version":"1.0","source":{"id":"2212.07919","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2212.07919","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"arxiv_version","alias_value":"2212.07919v2","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2212.07919","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_12","alias_value":"IXC5F7PDSFYD","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_16","alias_value":"IXC5F7PDSFYDQVS7","created_at":"2026-07-05T06:49:58Z"},{"alias_kind":"pith_short_8","alias_value":"IXC5F7PD","created_at":"2026-07-05T06:49:58Z"}],"graph_snapshots":[{"event_id":"sha256:b6225cf6bc74e9d04ba66afe2d5945ca38c4f8c9f0366ace8e1866232efb501c","target":"graph","created_at":"2026-07-05T06:49:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2212.07919/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models show improved downstream task performance when prompted to generate step-by-step reasoning to justify their final answers. These reasoning steps greatly improve model interpretability and verification, but objectively studying their correctness (independent of the final answer) is difficult without reliable methods for automatic evaluation. We simply do not know how often the stated reasoning steps actually support the final end task predictions. In this work, we present ROSCOE, a suite of interpretable, unsupervised automatic scores that improve and extend previous text ","authors_text":"Asli Celikyilmaz, Luke Zettlemoyer, Martin Corredor, Maryam Fazel-Zarandi, Moya Chen, Olga Golovneva, Spencer Poff","cross_cats":["cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T15:52:39Z","title":"ROSCOE: A Suite of Metrics for Scoring Step-by-Step Reasoning"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2212.07919","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:59cb5dc57eb9a30e6b4b5f6f9c6a444f1fef36eb31be74da68b305f172ec87fe","target":"record","created_at":"2026-07-05T06:49:58Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fb9af92dfca3e190b14c7d16e3b1baa668cd0a65acac7d79474cfa6ea5f82fdd","cross_cats_sorted":["cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-12-15T15:52:39Z","title_canon_sha256":"06f60e21f4c5e28452ac47ba5048b0184059ccc9bd4edb974b7fefc41ceff51e"},"schema_version":"1.0","source":{"id":"2212.07919","kind":"arxiv","version":2}},"canonical_sha256":"45c5d2fde3917038565f4399c8e61ce5a712815543618bf19c58ff55ebc23e76","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"45c5d2fde3917038565f4399c8e61ce5a712815543618bf19c58ff55ebc23e76","first_computed_at":"2026-07-05T06:49:58.595222Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:49:58.595222Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nPC9fQWSvCH9CFBk0abk3knmyZpTkYkK6G7V7TtdZWGC2JxQV6Fh6UJeDEdO7m6OFLAiWdpfuQt6KkBhJfnIAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:49:58.595849Z","signed_message":"canonical_sha256_bytes"},"source_id":"2212.07919","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:59cb5dc57eb9a30e6b4b5f6f9c6a444f1fef36eb31be74da68b305f172ec87fe","sha256:b6225cf6bc74e9d04ba66afe2d5945ca38c4f8c9f0366ace8e1866232efb501c"],"state_sha256":"5ec781b6a8cb49ca752115c9c646d6671cc8fb9c300074efe26e80b08df96ade"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"crXUqnBb3tiI8AMhS5QjcKQgUoOnfxqpCtNvEhzE7UshMHlhZqclVNmPtN8hdEX2BpR+EfTgdu19Om7Pb9C9Aw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T17:34:04.330116Z","bundle_sha256":"8b89aeaebb43751a933c32e93513032d453864b66aba1416f5c8653128613a4d"}}