{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:EZVLPNDGTSIHJJ7XVCJC7DBYGU","short_pith_number":"pith:EZVLPNDG","schema_version":"1.0","canonical_sha256":"266ab7b4669c9074a7f7a8922f8c38352ca5afa6bea797ef6ce8591e384b7354","source":{"kind":"arxiv","id":"2403.09032","version":3},"attestation_state":"computed","paper":{"title":"CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.SE","authors_text":"Aton Kamanda, Houari Sahraoui, Martin Weyssow, Xin Zhou","submitted_at":"2024-03-14T01:51:35Z","abstract_excerpt":"Evaluating the alignment of large language models (LLMs) with user-defined coding preferences is a challenging endeavour that requires a deep assessment of LLMs' outputs. Existing methods and benchmarks rely primarily on automated metrics and static analysis tools, which often fail to capture the nuances of user instructions and LLM outputs. To address this gap, we propose using the LLM-as-a-Judge methodology to evaluate the alignment of LLMs with coding preferences. Based on this approach, we present CodeUltraFeedback, a comprehensive dataset designed to facilitate the evaluation and improvem"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2403.09032","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2024-03-14T01:51:35Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"7d17dbce99de0574beaeb93e7d78dd22eee3a29495a9875644e3748829f19509","abstract_canon_sha256":"51d663df4581af78daa6d7b5da7a24549ce70784d2f84742a9a8c1cd5ce5f1bb"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:54:25.344506Z","signature_b64":"x1DCPxD8fubxvPHmc7rPz70YBAL5YI9HFhGsARzxdqzQb1hpIUnLgejefzdrjPcfw4YoBTkVHxwx6oTda1hqDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"266ab7b4669c9074a7f7a8922f8c38352ca5afa6bea797ef6ce8591e384b7354","last_reissued_at":"2026-07-05T09:54:25.343963Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:54:25.343963Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"CodeUltraFeedback: An LLM-as-a-Judge Dataset for Aligning Large Language Models to Coding Preferences","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.SE","authors_text":"Aton Kamanda, Houari Sahraoui, Martin Weyssow, Xin Zhou","submitted_at":"2024-03-14T01:51:35Z","abstract_excerpt":"Evaluating the alignment of large language models (LLMs) with user-defined coding preferences is a challenging endeavour that requires a deep assessment of LLMs' outputs. Existing methods and benchmarks rely primarily on automated metrics and static analysis tools, which often fail to capture the nuances of user instructions and LLM outputs. To address this gap, we propose using the LLM-as-a-Judge methodology to evaluate the alignment of LLMs with coding preferences. Based on this approach, we present CodeUltraFeedback, a comprehensive dataset designed to facilitate the evaluation and improvem"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.09032","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.09032/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2403.09032","created_at":"2026-07-05T09:54:25.344037+00:00"},{"alias_kind":"arxiv_version","alias_value":"2403.09032v3","created_at":"2026-07-05T09:54:25.344037+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.09032","created_at":"2026-07-05T09:54:25.344037+00:00"},{"alias_kind":"pith_short_12","alias_value":"EZVLPNDGTSIH","created_at":"2026-07-05T09:54:25.344037+00:00"},{"alias_kind":"pith_short_16","alias_value":"EZVLPNDGTSIHJJ7X","created_at":"2026-07-05T09:54:25.344037+00:00"},{"alias_kind":"pith_short_8","alias_value":"EZVLPNDG","created_at":"2026-07-05T09:54:25.344037+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.27028","citing_title":"Less is More: Early Stopping Rollout for On-Policy Distillation","ref_index":4,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14350","citing_title":"Distributionally Robust Multi-Task Reinforcement Learning via Adaptive Task Sampling","ref_index":248,"is_internal_anchor":false},{"citing_arxiv_id":"2412.05579","citing_title":"LLMs-as-Judges: A Comprehensive Survey on LLM-based Evaluation Methods","ref_index":252,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU","json":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU.json","graph_json":"https://pith.science/api/pith-number/EZVLPNDGTSIHJJ7XVCJC7DBYGU/graph.json","events_json":"https://pith.science/api/pith-number/EZVLPNDGTSIHJJ7XVCJC7DBYGU/events.json","paper":"https://pith.science/paper/EZVLPNDG"},"agent_actions":{"view_html":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU","download_json":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU.json","view_paper":"https://pith.science/paper/EZVLPNDG","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2403.09032&json=true","fetch_graph":"https://pith.science/api/pith-number/EZVLPNDGTSIHJJ7XVCJC7DBYGU/graph.json","fetch_events":"https://pith.science/api/pith-number/EZVLPNDGTSIHJJ7XVCJC7DBYGU/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU/action/timestamp_anchor","attest_storage":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU/action/storage_attestation","attest_author":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU/action/author_attestation","sign_citation":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU/action/citation_signature","submit_replication":"https://pith.science/pith/EZVLPNDGTSIHJJ7XVCJC7DBYGU/action/replication_record"}},"created_at":"2026-07-05T09:54:25.344037+00:00","updated_at":"2026-07-05T09:54:25.344037+00:00"}