{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:AA4FMG4RXCJ6YXBTZK7OVR4LFU","short_pith_number":"pith:AA4FMG4R","canonical_record":{"source":{"id":"2404.03602","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:19:47Z","cross_cats_sorted":[],"title_canon_sha256":"8457f26997419f545c95a8664ea98543a123ce6337159ec4b1a249d3a93406a4","abstract_canon_sha256":"a56c3ca30096ab2b619642fe128779b74fe8309f6a3d96c6f6c78066799c925e"},"schema_version":"1.0"},"canonical_sha256":"0038561b91b893ec5c33cabeeac78b2d081b0fccd0800e4f4894a6bc40e22e1a","source":{"kind":"arxiv","id":"2404.03602","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.03602","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"arxiv_version","alias_value":"2404.03602v2","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.03602","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"pith_short_12","alias_value":"AA4FMG4RXCJ6","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"pith_short_16","alias_value":"AA4FMG4RXCJ6YXBT","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"pith_short_8","alias_value":"AA4FMG4R","created_at":"2026-07-05T08:49:32Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:AA4FMG4RXCJ6YXBTZK7OVR4LFU","target":"record","payload":{"canonical_record":{"source":{"id":"2404.03602","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:19:47Z","cross_cats_sorted":[],"title_canon_sha256":"8457f26997419f545c95a8664ea98543a123ce6337159ec4b1a249d3a93406a4","abstract_canon_sha256":"a56c3ca30096ab2b619642fe128779b74fe8309f6a3d96c6f6c78066799c925e"},"schema_version":"1.0"},"canonical_sha256":"0038561b91b893ec5c33cabeeac78b2d081b0fccd0800e4f4894a6bc40e22e1a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:49:32.641327Z","signature_b64":"4DxhmIe0ffY7SE6qQJXGzMqeUtInl6VwKecsT8UWRXAiO4pb65+cfPON8RystSp8LhZ4VBtmP1cTYk6C+yNRCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0038561b91b893ec5c33cabeeac78b2d081b0fccd0800e4f4894a6bc40e22e1a","last_reissued_at":"2026-07-05T08:49:32.640838Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:49:32.640838Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.03602","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:49:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IYAV7VCGpbsFBB18Ygk4ZglxvSPm2otENKWIG6keH3xrsPgAuUIRBqaE2tvEsoQdVJgNzEWuHE9DC7AYEHMlAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:13:45.710273Z"},"content_sha256":"5537c1e3552f460184072bd42effed4c0f8502d4f706ae02d5648477cbd3e619","schema_version":"1.0","event_id":"sha256:5537c1e3552f460184072bd42effed4c0f8502d4f706ae02d5648477cbd3e619"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:AA4FMG4RXCJ6YXBTZK7OVR4LFU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Evaluating LLMs at Detecting Errors in LLM Responses","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Arman Cohan, Jihyun Janice Ahn, Nan Zhang, Ranran Haoran Zhang, Renze Lou, Rui Zhang, Ryo Kamoi, Salika Dave, Sarkar Snigdha Sarathi Das, Shaobo Qin, Sujeeth Reddy Vummanthala, Wenpeng Yin, Xiaoxin Lu, Yilun Zhao, Yusen Zhang","submitted_at":"2024-04-04T17:19:47Z","abstract_excerpt":"With Large Language Models (LLMs) being widely used across various tasks, detecting errors in their responses is increasingly crucial. However, little research has been conducted on error detection of LLM responses. Collecting error annotations on LLM responses is challenging due to the subjective nature of many NLP tasks, and thus previous research focuses on tasks of little practical value (e.g., word sorting) or limited error types (e.g., faithfulness in summarization). This work introduces ReaLMistake, the first error detection benchmark consisting of objective, realistic, and diverse erro"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.03602","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.03602/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:49:32Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iYzO3k2Vn9EJiSSXdNRxSsBD7W8geUr0XD+dM8hXrGILy3hF86b2xRkLKKlyN3T6oNNuMVqagsa0nz0VlgR2Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T13:13:45.710823Z"},"content_sha256":"3c9712200de91f724e2b6220268518c596015d23845275ba3a8e330c06623504","schema_version":"1.0","event_id":"sha256:3c9712200de91f724e2b6220268518c596015d23845275ba3a8e330c06623504"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/AA4FMG4RXCJ6YXBTZK7OVR4LFU/bundle.json","state_url":"https://pith.science/pith/AA4FMG4RXCJ6YXBTZK7OVR4LFU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/AA4FMG4RXCJ6YXBTZK7OVR4LFU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T13:13:45Z","links":{"resolver":"https://pith.science/pith/AA4FMG4RXCJ6YXBTZK7OVR4LFU","bundle":"https://pith.science/pith/AA4FMG4RXCJ6YXBTZK7OVR4LFU/bundle.json","state":"https://pith.science/pith/AA4FMG4RXCJ6YXBTZK7OVR4LFU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/AA4FMG4RXCJ6YXBTZK7OVR4LFU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:AA4FMG4RXCJ6YXBTZK7OVR4LFU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a56c3ca30096ab2b619642fe128779b74fe8309f6a3d96c6f6c78066799c925e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:19:47Z","title_canon_sha256":"8457f26997419f545c95a8664ea98543a123ce6337159ec4b1a249d3a93406a4"},"schema_version":"1.0","source":{"id":"2404.03602","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.03602","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"arxiv_version","alias_value":"2404.03602v2","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.03602","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"pith_short_12","alias_value":"AA4FMG4RXCJ6","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"pith_short_16","alias_value":"AA4FMG4RXCJ6YXBT","created_at":"2026-07-05T08:49:32Z"},{"alias_kind":"pith_short_8","alias_value":"AA4FMG4R","created_at":"2026-07-05T08:49:32Z"}],"graph_snapshots":[{"event_id":"sha256:3c9712200de91f724e2b6220268518c596015d23845275ba3a8e330c06623504","target":"graph","created_at":"2026-07-05T08:49:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.03602/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"With Large Language Models (LLMs) being widely used across various tasks, detecting errors in their responses is increasingly crucial. However, little research has been conducted on error detection of LLM responses. Collecting error annotations on LLM responses is challenging due to the subjective nature of many NLP tasks, and thus previous research focuses on tasks of little practical value (e.g., word sorting) or limited error types (e.g., faithfulness in summarization). This work introduces ReaLMistake, the first error detection benchmark consisting of objective, realistic, and diverse erro","authors_text":"Arman Cohan, Jihyun Janice Ahn, Nan Zhang, Ranran Haoran Zhang, Renze Lou, Rui Zhang, Ryo Kamoi, Salika Dave, Sarkar Snigdha Sarathi Das, Shaobo Qin, Sujeeth Reddy Vummanthala, Wenpeng Yin, Xiaoxin Lu, Yilun Zhao, Yusen Zhang","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:19:47Z","title":"Evaluating LLMs at Detecting Errors in LLM Responses"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.03602","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5537c1e3552f460184072bd42effed4c0f8502d4f706ae02d5648477cbd3e619","target":"record","created_at":"2026-07-05T08:49:32Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a56c3ca30096ab2b619642fe128779b74fe8309f6a3d96c6f6c78066799c925e","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-04-04T17:19:47Z","title_canon_sha256":"8457f26997419f545c95a8664ea98543a123ce6337159ec4b1a249d3a93406a4"},"schema_version":"1.0","source":{"id":"2404.03602","kind":"arxiv","version":2}},"canonical_sha256":"0038561b91b893ec5c33cabeeac78b2d081b0fccd0800e4f4894a6bc40e22e1a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0038561b91b893ec5c33cabeeac78b2d081b0fccd0800e4f4894a6bc40e22e1a","first_computed_at":"2026-07-05T08:49:32.640838Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:49:32.640838Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4DxhmIe0ffY7SE6qQJXGzMqeUtInl6VwKecsT8UWRXAiO4pb65+cfPON8RystSp8LhZ4VBtmP1cTYk6C+yNRCA==","signature_status":"signed_v1","signed_at":"2026-07-05T08:49:32.641327Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.03602","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5537c1e3552f460184072bd42effed4c0f8502d4f706ae02d5648477cbd3e619","sha256:3c9712200de91f724e2b6220268518c596015d23845275ba3a8e330c06623504"],"state_sha256":"2b51e5333f3d0578a4acc8227c1faecf0cdc2fecef517a02d2c8ad6ca6484055"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jDhVMHUOjXuhP2cA0QyM2tZ8n9iCKgnxGHybFYUAdONbnbf0bC7wc5mN3fRBM1qqUiIaOdc48+Ha5LGRfSd8CA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T13:13:45.716564Z","bundle_sha256":"2ffa07fb081de32df91864d533f63de6b18418ed397c344ed87efffe70b2161e"}}