{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:NJV77UEVZE5U3KALXXAU67TBBQ","short_pith_number":"pith:NJV77UEV","canonical_record":{"source":{"id":"2206.05802","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-12T17:40:53Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"4b18fe47a1e568196c4fcc15706c8ee66172cc44af3e8c8d09167fd731968593","abstract_canon_sha256":"0e4d7467e1bea5db6aa32a478e4f9eb5853b0a443eead77df48f927c6423d48a"},"schema_version":"1.0"},"canonical_sha256":"6a6bffd095c93b4da80bbdc14f7e610c2d31bcb63c215bcb6039ebcc42f59b49","source":{"kind":"arxiv","id":"2206.05802","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.05802","created_at":"2026-05-17T23:38:46Z"},{"alias_kind":"arxiv_version","alias_value":"2206.05802v2","created_at":"2026-05-17T23:38:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.05802","created_at":"2026-05-17T23:38:46Z"},{"alias_kind":"pith_short_12","alias_value":"NJV77UEVZE5U","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"NJV77UEVZE5U3KAL","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"NJV77UEV","created_at":"2026-05-18T12:33:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:NJV77UEVZE5U3KALXXAU67TBBQ","target":"record","payload":{"canonical_record":{"source":{"id":"2206.05802","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-12T17:40:53Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"4b18fe47a1e568196c4fcc15706c8ee66172cc44af3e8c8d09167fd731968593","abstract_canon_sha256":"0e4d7467e1bea5db6aa32a478e4f9eb5853b0a443eead77df48f927c6423d48a"},"schema_version":"1.0"},"canonical_sha256":"6a6bffd095c93b4da80bbdc14f7e610c2d31bcb63c215bcb6039ebcc42f59b49","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:46.693038Z","signature_b64":"3vVJrDHF31M9bRT3wDbsVeqcj4SnEr00DJH1GkGEK0gdo2RDQkIQqSDwOE4mPbet8ZTVA21UxAHeP8hmvBeXAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6a6bffd095c93b4da80bbdc14f7e610c2d31bcb63c215bcb6039ebcc42f59b49","last_reissued_at":"2026-05-17T23:38:46.692603Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:46.692603Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2206.05802","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:46Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wgQk7jK48zb4NxYdHb0pHpNEbRLk6n1X9cVRChTubvrvS+zx8K4MQ3M9FEVs6MT12L5xpQoSpquYMzF43Zt6AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-11T13:06:34.341225Z"},"content_sha256":"9fbd6150321a596d6255f8dda4471b09f8d48796007b0aca2012ea3dceb872f4","schema_version":"1.0","event_id":"sha256:9fbd6150321a596d6255f8dda4471b09f8d48796007b0aca2012ea3dceb872f4"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:NJV77UEVZE5U3KALXXAU67TBBQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Self-critiquing models for assisting human evaluators","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Fine-tuned language models can generate critiques that help humans identify flaws in summaries they would otherwise overlook.","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Catherine Yeh, Jan Leike, Jeff Wu, Jonathan Ward, Long Ouyang, Steven Bills, William Saunders","submitted_at":"2022-06-12T17:40:53Z","abstract_excerpt":"We fine-tune large language models to write natural language critiques (natural language critical comments) using behavioral cloning. On a topic-based summarization task, critiques written by our models help humans find flaws in summaries that they would have otherwise missed. Our models help find naturally occurring flaws in both model and human written summaries, and intentional flaws in summaries written by humans to be deliberately misleading. We study scaling properties of critiquing with both topic-based summarization and synthetic tasks. Larger models write more helpful critiques, and o"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Critiques written by our models help humans find flaws in summaries that they would have otherwise missed, including naturally occurring flaws in model and human written summaries and intentional flaws in deliberately misleading summaries.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the behavioral cloning from human critique examples produces critiques that generalize to new summaries without introducing systematic biases or missing important flaw types that humans would notice unaided.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"Fine-tuned LLMs produce critiques that improve human detection of errors in summaries, with larger models showing better self-critique and refinement capabilities.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Fine-tuned language models can generate critiques that help humans identify flaws in summaries they would otherwise overlook.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"835e3e6960dc71dd8db38a81eb031414b386104eb40e421cce5b8a25770bdba0"},"source":{"id":"2206.05802","kind":"arxiv","version":2},"verdict":{"id":"4679bf8b-4d3b-4bdd-aba7-1e6877298515","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-16T20:22:33.334217Z","strongest_claim":"Critiques written by our models help humans find flaws in summaries that they would have otherwise missed, including naturally occurring flaws in model and human written summaries and intentional flaws in deliberately misleading summaries.","one_line_summary":"Fine-tuned LLMs produce critiques that improve human detection of errors in summaries, with larger models showing better self-critique and refinement capabilities.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the behavioral cloning from human critique examples produces critiques that generalize to new summaries without introducing systematic biases or missing important flaw types that humans would notice unaided.","pith_extraction_headline":"Fine-tuned language models can generate critiques that help humans identify flaws in summaries they would otherwise overlook."},"references":{"count":22,"sample":[{"doi":"","year":null,"title":"We asked for labelers to aim to have answers with different kinds of flaws, e.g","work_id":"275e6586-c52e-4dc4-9514-30c821cc67db","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"quote corroborations","work_id":"e032ee9a-dcc3-4cf1-8b3a-83399b1df716","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"corroborations","work_id":"a8e3db18-e61a-4ae3-a93e-9d5693f04d44","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"For topic-based summarization, we asked for a category for each critique, one of: • Coverage: summary missing relevant information from passage • Accuracy: summary giving incorrect information • Coher","work_id":"67d66b02-c64e-4766-8a03-cfc52bb74268","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"passage highlights","work_id":"0de6b1cb-02f3-4f9b-a2b6-2681731be30c","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":22,"snapshot_sha256":"60954a31bcdbae34928f59d7e6b4831d6574d72ad3d1a9ed424d852180c88925","internal_anchors":0},"formal_canon":{"evidence_count":2,"snapshot_sha256":"9abf55d0055c45af03b330930d5ff1bf4ba17387d02536150204ee26b624388f"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"4679bf8b-4d3b-4bdd-aba7-1e6877298515"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:46Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pY21QrMfR3sdqynNzaNsRF8YtgWdOqGhN3wde6Q39amrDASEDlKWLgsZ+iYoxT3XEIG6qtJ667/qVPA3EcO/BA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-11T13:06:34.341871Z"},"content_sha256":"5273e47cef340ce1767c40364ffff53ce82400e7c156c6d2ddc059a7b5c66173","schema_version":"1.0","event_id":"sha256:5273e47cef340ce1767c40364ffff53ce82400e7c156c6d2ddc059a7b5c66173"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NJV77UEVZE5U3KALXXAU67TBBQ/bundle.json","state_url":"https://pith.science/pith/NJV77UEVZE5U3KALXXAU67TBBQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NJV77UEVZE5U3KALXXAU67TBBQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-11T13:06:34Z","links":{"resolver":"https://pith.science/pith/NJV77UEVZE5U3KALXXAU67TBBQ","bundle":"https://pith.science/pith/NJV77UEVZE5U3KALXXAU67TBBQ/bundle.json","state":"https://pith.science/pith/NJV77UEVZE5U3KALXXAU67TBBQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NJV77UEVZE5U3KALXXAU67TBBQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:NJV77UEVZE5U3KALXXAU67TBBQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0e4d7467e1bea5db6aa32a478e4f9eb5853b0a443eead77df48f927c6423d48a","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-12T17:40:53Z","title_canon_sha256":"4b18fe47a1e568196c4fcc15706c8ee66172cc44af3e8c8d09167fd731968593"},"schema_version":"1.0","source":{"id":"2206.05802","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.05802","created_at":"2026-05-17T23:38:46Z"},{"alias_kind":"arxiv_version","alias_value":"2206.05802v2","created_at":"2026-05-17T23:38:46Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.05802","created_at":"2026-05-17T23:38:46Z"},{"alias_kind":"pith_short_12","alias_value":"NJV77UEVZE5U","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"NJV77UEVZE5U3KAL","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"NJV77UEV","created_at":"2026-05-18T12:33:33Z"}],"graph_snapshots":[{"event_id":"sha256:5273e47cef340ce1767c40364ffff53ce82400e7c156c6d2ddc059a7b5c66173","target":"graph","created_at":"2026-05-17T23:38:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Critiques written by our models help humans find flaws in summaries that they would have otherwise missed, including naturally occurring flaws in model and human written summaries and intentional flaws in deliberately misleading summaries."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That the behavioral cloning from human critique examples produces critiques that generalize to new summaries without introducing systematic biases or missing important flaw types that humans would notice unaided."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"Fine-tuned LLMs produce critiques that improve human detection of errors in summaries, with larger models showing better self-critique and refinement capabilities."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Fine-tuned language models can generate critiques that help humans identify flaws in summaries they would otherwise overlook."}],"snapshot_sha256":"835e3e6960dc71dd8db38a81eb031414b386104eb40e421cce5b8a25770bdba0"},"formal_canon":{"evidence_count":2,"snapshot_sha256":"9abf55d0055c45af03b330930d5ff1bf4ba17387d02536150204ee26b624388f"},"paper":{"abstract_excerpt":"We fine-tune large language models to write natural language critiques (natural language critical comments) using behavioral cloning. On a topic-based summarization task, critiques written by our models help humans find flaws in summaries that they would have otherwise missed. Our models help find naturally occurring flaws in both model and human written summaries, and intentional flaws in summaries written by humans to be deliberately misleading. We study scaling properties of critiquing with both topic-based summarization and synthetic tasks. Larger models write more helpful critiques, and o","authors_text":"Catherine Yeh, Jan Leike, Jeff Wu, Jonathan Ward, Long Ouyang, Steven Bills, William Saunders","cross_cats":["cs.LG"],"headline":"Fine-tuned language models can generate critiques that help humans identify flaws in summaries they would otherwise overlook.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-12T17:40:53Z","title":"Self-critiquing models for assisting human evaluators"},"references":{"count":22,"internal_anchors":0,"resolved_work":22,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"We asked for labelers to aim to have answers with different kinds of flaws, e.g","work_id":"275e6586-c52e-4dc4-9514-30c821cc67db","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"quote corroborations","work_id":"e032ee9a-dcc3-4cf1-8b3a-83399b1df716","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":3,"title":"corroborations","work_id":"a8e3db18-e61a-4ae3-a93e-9d5693f04d44","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"For topic-based summarization, we asked for a category for each critique, one of: • Coverage: summary missing relevant information from passage • Accuracy: summary giving incorrect information • Coher","work_id":"67d66b02-c64e-4766-8a03-cfc52bb74268","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"passage highlights","work_id":"0de6b1cb-02f3-4f9b-a2b6-2681731be30c","year":null}],"snapshot_sha256":"60954a31bcdbae34928f59d7e6b4831d6574d72ad3d1a9ed424d852180c88925"},"source":{"id":"2206.05802","kind":"arxiv","version":2},"verdict":{"created_at":"2026-05-16T20:22:33.334217Z","id":"4679bf8b-4d3b-4bdd-aba7-1e6877298515","model_set":{"reader":"grok-4.3"},"one_line_summary":"Fine-tuned LLMs produce critiques that improve human detection of errors in summaries, with larger models showing better self-critique and refinement capabilities.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Fine-tuned language models can generate critiques that help humans identify flaws in summaries they would otherwise overlook.","strongest_claim":"Critiques written by our models help humans find flaws in summaries that they would have otherwise missed, including naturally occurring flaws in model and human written summaries and intentional flaws in deliberately misleading summaries.","weakest_assumption":"That the behavioral cloning from human critique examples produces critiques that generalize to new summaries without introducing systematic biases or missing important flaw types that humans would notice unaided."}},"verdict_id":"4679bf8b-4d3b-4bdd-aba7-1e6877298515"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9fbd6150321a596d6255f8dda4471b09f8d48796007b0aca2012ea3dceb872f4","target":"record","created_at":"2026-05-17T23:38:46Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0e4d7467e1bea5db6aa32a478e4f9eb5853b0a443eead77df48f927c6423d48a","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-12T17:40:53Z","title_canon_sha256":"4b18fe47a1e568196c4fcc15706c8ee66172cc44af3e8c8d09167fd731968593"},"schema_version":"1.0","source":{"id":"2206.05802","kind":"arxiv","version":2}},"canonical_sha256":"6a6bffd095c93b4da80bbdc14f7e610c2d31bcb63c215bcb6039ebcc42f59b49","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6a6bffd095c93b4da80bbdc14f7e610c2d31bcb63c215bcb6039ebcc42f59b49","first_computed_at":"2026-05-17T23:38:46.692603Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:46.692603Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3vVJrDHF31M9bRT3wDbsVeqcj4SnEr00DJH1GkGEK0gdo2RDQkIQqSDwOE4mPbet8ZTVA21UxAHeP8hmvBeXAQ==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:46.693038Z","signed_message":"canonical_sha256_bytes"},"source_id":"2206.05802","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9fbd6150321a596d6255f8dda4471b09f8d48796007b0aca2012ea3dceb872f4","sha256:5273e47cef340ce1767c40364ffff53ce82400e7c156c6d2ddc059a7b5c66173"],"state_sha256":"a8b7c28b9447347cfcc2ec2ab871fcd86c7e1b55dd10add0abd82d8a0a92fb5e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DeVh9TVVBf+uks1iTiJZQRle2bGqgMrfTaN1Q5YYGhjL+6taeKh6rDb9ziN8aPpBz2f52KOGdmjC3JCRrvZuDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-11T13:06:34.344536Z","bundle_sha256":"f0f24a1c575dada75d3b1f884f1d642f967782db69ba560725bc9633da913c54"}}