{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:MOAOMTMUS46UGXVB5VMLGUZ7JZ","short_pith_number":"pith:MOAOMTMU","canonical_record":{"source":{"id":"2401.12873","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-23T16:07:43Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"96f3772bdfd91f57a29d3e8ce187479417a0f27db615ec96965836613a7e21da","abstract_canon_sha256":"54fd3b143d3570d4752cc13f9b94dae8b308d3784e14490711d5b8962d06e98b"},"schema_version":"1.0"},"canonical_sha256":"6380e64d94973d435ea1ed58b3533f4e4b7e4f471480ee533c821a7a623684c2","source":{"kind":"arxiv","id":"2401.12873","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.12873","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"arxiv_version","alias_value":"2401.12873v3","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.12873","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"pith_short_12","alias_value":"MOAOMTMUS46U","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"pith_short_16","alias_value":"MOAOMTMUS46UGXVB","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"pith_short_8","alias_value":"MOAOMTMU","created_at":"2026-07-05T07:57:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:MOAOMTMUS46UGXVB5VMLGUZ7JZ","target":"record","payload":{"canonical_record":{"source":{"id":"2401.12873","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-23T16:07:43Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"96f3772bdfd91f57a29d3e8ce187479417a0f27db615ec96965836613a7e21da","abstract_canon_sha256":"54fd3b143d3570d4752cc13f9b94dae8b308d3784e14490711d5b8962d06e98b"},"schema_version":"1.0"},"canonical_sha256":"6380e64d94973d435ea1ed58b3533f4e4b7e4f471480ee533c821a7a623684c2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:57:29.891306Z","signature_b64":"A3/w7ICX6i05zp8cqfqvk6h34bJalTHDDDi8cUsecOZ6e6tCMADwliOrm9MEGO5HmU9RUJLosvEmrlIY/YboCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6380e64d94973d435ea1ed58b3533f4e4b7e4f471480ee533c821a7a623684c2","last_reissued_at":"2026-07-05T07:57:29.890680Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:57:29.890680Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2401.12873","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:57:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ves8uYvV+Hqq4vVUKaZ/o+OHbbNa/6HS+WzLju1zjvbpEvFzpjB1HbkrXrumTJPDvWodEJSRPpkHXA1m2TJzCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T17:15:54.813825Z"},"content_sha256":"1deda9ef6cfec7af66392a38ebb4c06d21c5395e1212582006fdd39abe5e80f3","schema_version":"1.0","event_id":"sha256:1deda9ef6cfec7af66392a38ebb4c06d21c5395e1212582006fdd39abe5e80f3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:MOAOMTMUS46UGXVB5VMLGUZ7JZ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Improving Machine Translation with Human Feedback: An Exploration of Quality Estimation as a Reward Model","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Rui Wang, Shuming Shi, Wenxiang Jiao, Xing Wang, Zhaopeng Tu, Zhiwei He, Zhuosheng Zhang","submitted_at":"2024-01-23T16:07:43Z","abstract_excerpt":"Insufficient modeling of human preferences within the reward model is a major obstacle for leveraging human feedback to improve translation quality. Fortunately, quality estimation (QE), which predicts the quality of a given translation without reference, has achieved impressive alignment with human evaluations in the last two years. In this work, we investigate the potential of employing the QE model as the reward model to predict human preferences for feedback training. We first identify the overoptimization problem during QE-based feedback training, manifested as an increase in reward while"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.12873","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2401.12873/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:57:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WUni4BRcndsgx4OezKmPM0hZSGDafNyT9mONnwTyWyOvsBjOysWEM0YibtlKDDNhS6PGj3nzwW97cCA2N5Z4Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T17:15:54.814335Z"},"content_sha256":"238fccc5449875d563f74ee35d9cf152793d6995395714a8ae027b7d300adf93","schema_version":"1.0","event_id":"sha256:238fccc5449875d563f74ee35d9cf152793d6995395714a8ae027b7d300adf93"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/MOAOMTMUS46UGXVB5VMLGUZ7JZ/bundle.json","state_url":"https://pith.science/pith/MOAOMTMUS46UGXVB5VMLGUZ7JZ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/MOAOMTMUS46UGXVB5VMLGUZ7JZ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T17:15:54Z","links":{"resolver":"https://pith.science/pith/MOAOMTMUS46UGXVB5VMLGUZ7JZ","bundle":"https://pith.science/pith/MOAOMTMUS46UGXVB5VMLGUZ7JZ/bundle.json","state":"https://pith.science/pith/MOAOMTMUS46UGXVB5VMLGUZ7JZ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/MOAOMTMUS46UGXVB5VMLGUZ7JZ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:MOAOMTMUS46UGXVB5VMLGUZ7JZ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"54fd3b143d3570d4752cc13f9b94dae8b308d3784e14490711d5b8962d06e98b","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-23T16:07:43Z","title_canon_sha256":"96f3772bdfd91f57a29d3e8ce187479417a0f27db615ec96965836613a7e21da"},"schema_version":"1.0","source":{"id":"2401.12873","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2401.12873","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"arxiv_version","alias_value":"2401.12873v3","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2401.12873","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"pith_short_12","alias_value":"MOAOMTMUS46U","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"pith_short_16","alias_value":"MOAOMTMUS46UGXVB","created_at":"2026-07-05T07:57:29Z"},{"alias_kind":"pith_short_8","alias_value":"MOAOMTMU","created_at":"2026-07-05T07:57:29Z"}],"graph_snapshots":[{"event_id":"sha256:238fccc5449875d563f74ee35d9cf152793d6995395714a8ae027b7d300adf93","target":"graph","created_at":"2026-07-05T07:57:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2401.12873/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Insufficient modeling of human preferences within the reward model is a major obstacle for leveraging human feedback to improve translation quality. Fortunately, quality estimation (QE), which predicts the quality of a given translation without reference, has achieved impressive alignment with human evaluations in the last two years. In this work, we investigate the potential of employing the QE model as the reward model to predict human preferences for feedback training. We first identify the overoptimization problem during QE-based feedback training, manifested as an increase in reward while","authors_text":"Rui Wang, Shuming Shi, Wenxiang Jiao, Xing Wang, Zhaopeng Tu, Zhiwei He, Zhuosheng Zhang","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-23T16:07:43Z","title":"Improving Machine Translation with Human Feedback: An Exploration of Quality Estimation as a Reward Model"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2401.12873","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:1deda9ef6cfec7af66392a38ebb4c06d21c5395e1212582006fdd39abe5e80f3","target":"record","created_at":"2026-07-05T07:57:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"54fd3b143d3570d4752cc13f9b94dae8b308d3784e14490711d5b8962d06e98b","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2024-01-23T16:07:43Z","title_canon_sha256":"96f3772bdfd91f57a29d3e8ce187479417a0f27db615ec96965836613a7e21da"},"schema_version":"1.0","source":{"id":"2401.12873","kind":"arxiv","version":3}},"canonical_sha256":"6380e64d94973d435ea1ed58b3533f4e4b7e4f471480ee533c821a7a623684c2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6380e64d94973d435ea1ed58b3533f4e4b7e4f471480ee533c821a7a623684c2","first_computed_at":"2026-07-05T07:57:29.890680Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:57:29.890680Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"A3/w7ICX6i05zp8cqfqvk6h34bJalTHDDDi8cUsecOZ6e6tCMADwliOrm9MEGO5HmU9RUJLosvEmrlIY/YboCA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:57:29.891306Z","signed_message":"canonical_sha256_bytes"},"source_id":"2401.12873","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:1deda9ef6cfec7af66392a38ebb4c06d21c5395e1212582006fdd39abe5e80f3","sha256:238fccc5449875d563f74ee35d9cf152793d6995395714a8ae027b7d300adf93"],"state_sha256":"74d1da84f8ed8459f9acff1bfc51fe09ac77572b398ac4c2f518a08fd5421f82"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"G9x/lCrgZ9Jb0lr8Ls/W0iU8kb/vYt2YWMO+k2+YmqEGei0gm4oyQPTXmA4zPuGrrFbyLlGosxaK5FGcxxEdCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T17:15:54.819749Z","bundle_sha256":"c1c2111665c7bbd566844b91c33e7e1257d64165da17b720772805de9d6ad81f"}}