{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:MU77BDGIT2OL5HJMFONZARMFMM","short_pith_number":"pith:MU77BDGI","schema_version":"1.0","canonical_sha256":"653ff08cc89e9cbe9d2c2b9b90458563244ec61c17f580b2df133bc3ec1d8815","source":{"kind":"arxiv","id":"2408.02666","version":2},"attestation_state":"computed","paper":{"title":"Self-Taught Evaluators","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ilia Kulikov, Jane Dwivedi-Yu, Jason Weston, Maryam Fazel-Zarandi, Olga Golovneva, Ping Yu, Richard Yuanzhe Pang, Tianlu Wang, Weizhe Yuan, Xian Li","submitted_at":"2024-08-05T17:57:02Z","abstract_excerpt":"Model-based evaluation is at the heart of successful model development -- as a reward model for training, and as a replacement for human evaluation. To train such evaluators, the standard approach is to collect a large amount of human preference judgments over model responses, which is costly and the data becomes stale as models improve. In this work, we present an approach that aims to im-prove evaluators without human annotations, using synthetic training data only. Starting from unlabeled instructions, our iterative self-improvement scheme generates contrasting model outputs and trains an L"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.02666","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/publicdomain/zero/1.0/","primary_cat":"cs.CL","submitted_at":"2024-08-05T17:57:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"61e0375e3ac76df116e841e157a569167dec68517e09937887ac56feada5b939","abstract_canon_sha256":"0e5f3a4bd6e624992f6f59fce015e3b3d2e5c7a5164b169ffad847fb69f6a539"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:53:25.050782Z","signature_b64":"yHcSQAOlYQ7YfXnHg3OktNCZiZ0QWWoFDam5C5v7xOehCjhU2HGR/7RfsqObWSMkAR1BS7RdlR6rf00ll8vyDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"653ff08cc89e9cbe9d2c2b9b90458563244ec61c17f580b2df133bc3ec1d8815","last_reissued_at":"2026-07-05T08:53:25.050366Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:53:25.050366Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Self-Taught Evaluators","license":"http://creativecommons.org/publicdomain/zero/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Ilia Kulikov, Jane Dwivedi-Yu, Jason Weston, Maryam Fazel-Zarandi, Olga Golovneva, Ping Yu, Richard Yuanzhe Pang, Tianlu Wang, Weizhe Yuan, Xian Li","submitted_at":"2024-08-05T17:57:02Z","abstract_excerpt":"Model-based evaluation is at the heart of successful model development -- as a reward model for training, and as a replacement for human evaluation. To train such evaluators, the standard approach is to collect a large amount of human preference judgments over model responses, which is costly and the data becomes stale as models improve. In this work, we present an approach that aims to im-prove evaluators without human annotations, using synthetic training data only. Starting from unlabeled instructions, our iterative self-improvement scheme generates contrasting model outputs and trains an L"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.02666","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.02666/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.02666","created_at":"2026-07-05T08:53:25.050420+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.02666v2","created_at":"2026-07-05T08:53:25.050420+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.02666","created_at":"2026-07-05T08:53:25.050420+00:00"},{"alias_kind":"pith_short_12","alias_value":"MU77BDGIT2OL","created_at":"2026-07-05T08:53:25.050420+00:00"},{"alias_kind":"pith_short_16","alias_value":"MU77BDGIT2OL5HJM","created_at":"2026-07-05T08:53:25.050420+00:00"},{"alias_kind":"pith_short_8","alias_value":"MU77BDGI","created_at":"2026-07-05T08:53:25.050420+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":9,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.24004","citing_title":"Towards Spec Learning: Inference-Time Alignment from Preference Pairs","ref_index":105,"is_internal_anchor":false},{"citing_arxiv_id":"2606.24004","citing_title":"Towards Spec Learning: Inference-Time Alignment from Preference Pairs","ref_index":105,"is_internal_anchor":false},{"citing_arxiv_id":"2606.28898","citing_title":"PASTA: A Paraphrasing And Self-Training Approach for Knowledge Updating in LLMs","ref_index":24,"is_internal_anchor":false},{"citing_arxiv_id":"2411.15594","citing_title":"A Survey on LLM-as-a-Judge","ref_index":162,"is_internal_anchor":false},{"citing_arxiv_id":"2509.23542","citing_title":"On the Shelf Life of Fine-Tuned LLM-Judges: Future-Proofing, Backward-Compatibility, and Question Generalization","ref_index":44,"is_internal_anchor":false},{"citing_arxiv_id":"2601.21464","citing_title":"Conversation for Non-verifiable Learning: Self-Evolving LLMs through Meta-Evaluation","ref_index":22,"is_internal_anchor":false},{"citing_arxiv_id":"2412.05579","citing_title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","ref_index":240,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04542","citing_title":"Power Distribution Bridges Sampling, Self-Reward RL, and Self-Distillation","ref_index":97,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01687","citing_title":"MultiBreak: A Scalable and Diverse Multi-turn Jailbreak Benchmark for Evaluating LLM Safety","ref_index":46,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM","json":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM.json","graph_json":"https://pith.science/api/pith-number/MU77BDGIT2OL5HJMFONZARMFMM/graph.json","events_json":"https://pith.science/api/pith-number/MU77BDGIT2OL5HJMFONZARMFMM/events.json","paper":"https://pith.science/paper/MU77BDGI"},"agent_actions":{"view_html":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM","download_json":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM.json","view_paper":"https://pith.science/paper/MU77BDGI","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.02666&json=true","fetch_graph":"https://pith.science/api/pith-number/MU77BDGIT2OL5HJMFONZARMFMM/graph.json","fetch_events":"https://pith.science/api/pith-number/MU77BDGIT2OL5HJMFONZARMFMM/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM/action/timestamp_anchor","attest_storage":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM/action/storage_attestation","attest_author":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM/action/author_attestation","sign_citation":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM/action/citation_signature","submit_replication":"https://pith.science/pith/MU77BDGIT2OL5HJMFONZARMFMM/action/replication_record"}},"created_at":"2026-07-05T08:53:25.050420+00:00","updated_at":"2026-07-05T08:53:25.050420+00:00"}