{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:P6QICAVFFCJGHS7ZUS2Y44XVTV","short_pith_number":"pith:P6QICAVF","schema_version":"1.0","canonical_sha256":"7fa08102a5289263cbf9a4b58e72f59d45cf6b999182d9c13d80d3fe19127e65","source":{"kind":"arxiv","id":"2208.03133","version":2},"attestation_state":"computed","paper":{"title":"Out of the BLEU: how should we assess quality of the Code Generation models?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.SE","authors_text":"Egor Bogomolov, Mikhail Evtikhiev, Timofey Bryksin, Yaroslav Sokolov","submitted_at":"2022-08-05T13:00:16Z","abstract_excerpt":"In recent years, researchers have created and introduced a significant number of various code generation models. As human evaluation of every new model version is unfeasible, the community adopted automatic evaluation metrics such as BLEU to approximate the results of human judgement. These metrics originate from the machine translation domain and it is unclear whether they are applicable for the code generation tasks and how well they agree with the human evaluation on this task. There are also other metrics, CodeBLEU and RUBY, developed to estimate the similarity of code, that take into acco"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2208.03133","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2022-08-05T13:00:16Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"f326d28db7f5a578d308645e79bf2d67cb8299e4b9472eaa20dd70500d5cd6bc","abstract_canon_sha256":"18fbb516a3e0e3f8b3ab87c706c0a21735f91a7f15a3833c917e1c9a8087197a"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:08:51.738639Z","signature_b64":"hi5/9KHi8ZO1CJ/sZxnTRVxh3dl1UVLBKixjE5qtYaE9skNsPS9Vs5w7VjC7u2k5Nr1h0NfV8Ug7qZExe4tGAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7fa08102a5289263cbf9a4b58e72f59d45cf6b999182d9c13d80d3fe19127e65","last_reissued_at":"2026-07-05T06:08:51.738170Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:08:51.738170Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Out of the BLEU: how should we assess quality of the Code Generation models?","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.SE","authors_text":"Egor Bogomolov, Mikhail Evtikhiev, Timofey Bryksin, Yaroslav Sokolov","submitted_at":"2022-08-05T13:00:16Z","abstract_excerpt":"In recent years, researchers have created and introduced a significant number of various code generation models. As human evaluation of every new model version is unfeasible, the community adopted automatic evaluation metrics such as BLEU to approximate the results of human judgement. These metrics originate from the machine translation domain and it is unclear whether they are applicable for the code generation tasks and how well they agree with the human evaluation on this task. There are also other metrics, CodeBLEU and RUBY, developed to estimate the similarity of code, that take into acco"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2208.03133","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2208.03133/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2208.03133","created_at":"2026-07-05T06:08:51.738230+00:00"},{"alias_kind":"arxiv_version","alias_value":"2208.03133v2","created_at":"2026-07-05T06:08:51.738230+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2208.03133","created_at":"2026-07-05T06:08:51.738230+00:00"},{"alias_kind":"pith_short_12","alias_value":"P6QICAVFFCJG","created_at":"2026-07-05T06:08:51.738230+00:00"},{"alias_kind":"pith_short_16","alias_value":"P6QICAVFFCJGHS7Z","created_at":"2026-07-05T06:08:51.738230+00:00"},{"alias_kind":"pith_short_8","alias_value":"P6QICAVF","created_at":"2026-07-05T06:08:51.738230+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.04273","citing_title":"Characterizing initial human-AI proof formalization workflows","ref_index":125,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV","json":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV.json","graph_json":"https://pith.science/api/pith-number/P6QICAVFFCJGHS7ZUS2Y44XVTV/graph.json","events_json":"https://pith.science/api/pith-number/P6QICAVFFCJGHS7ZUS2Y44XVTV/events.json","paper":"https://pith.science/paper/P6QICAVF"},"agent_actions":{"view_html":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV","download_json":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV.json","view_paper":"https://pith.science/paper/P6QICAVF","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2208.03133&json=true","fetch_graph":"https://pith.science/api/pith-number/P6QICAVFFCJGHS7ZUS2Y44XVTV/graph.json","fetch_events":"https://pith.science/api/pith-number/P6QICAVFFCJGHS7ZUS2Y44XVTV/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV/action/timestamp_anchor","attest_storage":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV/action/storage_attestation","attest_author":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV/action/author_attestation","sign_citation":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV/action/citation_signature","submit_replication":"https://pith.science/pith/P6QICAVFFCJGHS7ZUS2Y44XVTV/action/replication_record"}},"created_at":"2026-07-05T06:08:51.738230+00:00","updated_at":"2026-07-05T06:08:51.738230+00:00"}