{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:D46ZJVGFQDVXOP2IMPADQSJTST","short_pith_number":"pith:D46ZJVGF","canonical_record":{"source":{"id":"2405.20850","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-31T14:33:07Z","cross_cats_sorted":[],"title_canon_sha256":"462ee7d9e8d357616d0c3c2835c483346b3f52af7112b0fd1f99063b08dde254","abstract_canon_sha256":"a43d91a057520fc57a4ad9c144214916a19f4d4dfb355dc8eb353a60793b3e3c"},"schema_version":"1.0"},"canonical_sha256":"1f3d94d4c580eb773f4863c038493394ececa05cb8401cb63f33648d0d70a157","source":{"kind":"arxiv","id":"2405.20850","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.20850","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"arxiv_version","alias_value":"2405.20850v2","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.20850","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_12","alias_value":"D46ZJVGFQDVX","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_16","alias_value":"D46ZJVGFQDVXOP2I","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_8","alias_value":"D46ZJVGF","created_at":"2026-07-05T09:22:18Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:D46ZJVGFQDVXOP2IMPADQSJTST","target":"record","payload":{"canonical_record":{"source":{"id":"2405.20850","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-31T14:33:07Z","cross_cats_sorted":[],"title_canon_sha256":"462ee7d9e8d357616d0c3c2835c483346b3f52af7112b0fd1f99063b08dde254","abstract_canon_sha256":"a43d91a057520fc57a4ad9c144214916a19f4d4dfb355dc8eb353a60793b3e3c"},"schema_version":"1.0"},"canonical_sha256":"1f3d94d4c580eb773f4863c038493394ececa05cb8401cb63f33648d0d70a157","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:22:18.651541Z","signature_b64":"AjWw395EcmaCmfrt6ouX9N2XacfaVQr6jdehu18YnH5t8KZWDdTPLPHG8aT65JH0jutBcah8cfMNaq8fHrj4DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1f3d94d4c580eb773f4863c038493394ececa05cb8401cb63f33648d0d70a157","last_reissued_at":"2026-07-05T09:22:18.651045Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:22:18.651045Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.20850","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:22:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7Up6fiVn18LmaveBTGJAipT3R1/VCAJlYmNmKg/RdJZJ9MFeOFJiC7eyvoiwDjEu54yy1r7hcX2YaOh7aO4LBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T07:37:47.965196Z"},"content_sha256":"8a7dfc2e9a828b034b292eb2d888242fe8a89d1d771ecce6fbc3cc2fde13af17","schema_version":"1.0","event_id":"sha256:8a7dfc2e9a828b034b292eb2d888242fe8a89d1d771ecce6fbc3cc2fde13af17"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:D46ZJVGFQDVXOP2IMPADQSJTST","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Improving Reward Models with Synthetic Critiques","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Fraser Greenlee-Scott, Jon Ander Campos, Matthias Gall\\'e, Max Bartolo, Phil Blunsom, Zihuiwen Ye","submitted_at":"2024-05-31T14:33:07Z","abstract_excerpt":"Reward models (RMs) play a critical role in aligning language models through the process of reinforcement learning from human feedback. RMs are trained to predict a score reflecting human preference, which requires significant time and cost for human annotation. Additionally, RMs tend to quickly overfit on superficial features in the training set, hindering their generalization performance on unseen distributions. We propose a novel approach using synthetic natural language critiques generated by large language models to provide additional feedback, evaluating aspects such as instruction follo"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.20850","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.20850/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:22:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"t7qpdfp4FkH8vBFTf01fG3axRBVXzu6XuquYx7ukM5LM2syNgMto97FKNWYhb2oHbKY3gW+bqXm3eZSCaRzKDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T07:37:47.965700Z"},"content_sha256":"3abb6639326720dc2d327d1629f8d357487190bdcca61be93be829796c5187b5","schema_version":"1.0","event_id":"sha256:3abb6639326720dc2d327d1629f8d357487190bdcca61be93be829796c5187b5"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/D46ZJVGFQDVXOP2IMPADQSJTST/bundle.json","state_url":"https://pith.science/pith/D46ZJVGFQDVXOP2IMPADQSJTST/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/D46ZJVGFQDVXOP2IMPADQSJTST/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T07:37:47Z","links":{"resolver":"https://pith.science/pith/D46ZJVGFQDVXOP2IMPADQSJTST","bundle":"https://pith.science/pith/D46ZJVGFQDVXOP2IMPADQSJTST/bundle.json","state":"https://pith.science/pith/D46ZJVGFQDVXOP2IMPADQSJTST/state.json","well_known_bundle":"https://pith.science/.well-known/pith/D46ZJVGFQDVXOP2IMPADQSJTST/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:D46ZJVGFQDVXOP2IMPADQSJTST","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a43d91a057520fc57a4ad9c144214916a19f4d4dfb355dc8eb353a60793b3e3c","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-31T14:33:07Z","title_canon_sha256":"462ee7d9e8d357616d0c3c2835c483346b3f52af7112b0fd1f99063b08dde254"},"schema_version":"1.0","source":{"id":"2405.20850","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.20850","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"arxiv_version","alias_value":"2405.20850v2","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.20850","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_12","alias_value":"D46ZJVGFQDVX","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_16","alias_value":"D46ZJVGFQDVXOP2I","created_at":"2026-07-05T09:22:18Z"},{"alias_kind":"pith_short_8","alias_value":"D46ZJVGF","created_at":"2026-07-05T09:22:18Z"}],"graph_snapshots":[{"event_id":"sha256:3abb6639326720dc2d327d1629f8d357487190bdcca61be93be829796c5187b5","target":"graph","created_at":"2026-07-05T09:22:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.20850/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reward models (RMs) play a critical role in aligning language models through the process of reinforcement learning from human feedback. RMs are trained to predict a score reflecting human preference, which requires significant time and cost for human annotation. Additionally, RMs tend to quickly overfit on superficial features in the training set, hindering their generalization performance on unseen distributions. We propose a novel approach using synthetic natural language critiques generated by large language models to provide additional feedback, evaluating aspects such as instruction follo","authors_text":"Fraser Greenlee-Scott, Jon Ander Campos, Matthias Gall\\'e, Max Bartolo, Phil Blunsom, Zihuiwen Ye","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-31T14:33:07Z","title":"Improving Reward Models with Synthetic Critiques"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.20850","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8a7dfc2e9a828b034b292eb2d888242fe8a89d1d771ecce6fbc3cc2fde13af17","target":"record","created_at":"2026-07-05T09:22:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a43d91a057520fc57a4ad9c144214916a19f4d4dfb355dc8eb353a60793b3e3c","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-31T14:33:07Z","title_canon_sha256":"462ee7d9e8d357616d0c3c2835c483346b3f52af7112b0fd1f99063b08dde254"},"schema_version":"1.0","source":{"id":"2405.20850","kind":"arxiv","version":2}},"canonical_sha256":"1f3d94d4c580eb773f4863c038493394ececa05cb8401cb63f33648d0d70a157","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1f3d94d4c580eb773f4863c038493394ececa05cb8401cb63f33648d0d70a157","first_computed_at":"2026-07-05T09:22:18.651045Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:22:18.651045Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"AjWw395EcmaCmfrt6ouX9N2XacfaVQr6jdehu18YnH5t8KZWDdTPLPHG8aT65JH0jutBcah8cfMNaq8fHrj4DA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:22:18.651541Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.20850","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8a7dfc2e9a828b034b292eb2d888242fe8a89d1d771ecce6fbc3cc2fde13af17","sha256:3abb6639326720dc2d327d1629f8d357487190bdcca61be93be829796c5187b5"],"state_sha256":"77b4356ceedb4f932309f4359ea43e1f0fd1013e5ecfd972930bb02efa411ded"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CMQnjbnn55+Eu4oNNty5pKAmtLJtil7QgZ3bxiGisgpGVC7ZoS2341AltPZF0T2mkHI7B+9lTx1Fv1BYsXrpCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T07:37:47.971223Z","bundle_sha256":"8c481c7da747dd3ff3ed80f971ae8ec851cbe8194f28a15e60ba2ac5bf319f08"}}