{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:T547BO7OQXXKM4QPCFBGCTUIZ3","short_pith_number":"pith:T547BO7O","schema_version":"1.0","canonical_sha256":"9f79f0bbee85eea6720f1142614e88ced03d568a5a40c6894b559ddcaa2081f4","source":{"kind":"arxiv","id":"2410.12046","version":2},"attestation_state":"computed","paper":{"title":"Towards Realistic Evaluation of Commit Message Generation by Matching Online and Offline Settings","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.HC","cs.LG"],"primary_cat":"cs.SE","authors_text":"Aleksandra Eliseeva, Alexander Bezzubov, Danny Dig, Petr Tsvetkov, Timofey Bryksin, Yaroslav Golubev, Yaroslav Zharov","submitted_at":"2024-10-15T20:32:07Z","abstract_excerpt":"When a Commit Message Generation (CMG) system is integrated into the IDEs and other products at JetBrains, we perform online evaluation based on user acceptance of the generated messages. However, performing online experiments with every change to a CMG system is troublesome, as each iteration affects users and requires time to collect enough statistics. On the other hand, offline evaluation, a prevalent approach in the research literature, facilitates fast experiments but employs automatic metrics that are not guaranteed to represent the preferences of real users. In this work, we describe a "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2410.12046","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SE","submitted_at":"2024-10-15T20:32:07Z","cross_cats_sorted":["cs.HC","cs.LG"],"title_canon_sha256":"74681666c34b963d5c4607f53c60c3d2cdf8fb7f87f600110b20ff2265cc9f40","abstract_canon_sha256":"25a45dffdbf7c90f72b2fb80e74f7d39dc760bd5dc42c4d2dc924b263abf6ef7"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:58:25.373830Z","signature_b64":"SZgY04tW+iQLUmshIBYpAYmKd7rg5s9CKkTqxKgsX1c1RiXq8qfqbkEWtzxpB0FbutuGpDf2H2Q8EZN2NxS8BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9f79f0bbee85eea6720f1142614e88ced03d568a5a40c6894b559ddcaa2081f4","last_reissued_at":"2026-07-05T09:58:25.373252Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:58:25.373252Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Towards Realistic Evaluation of Commit Message Generation by Matching Online and Offline Settings","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.HC","cs.LG"],"primary_cat":"cs.SE","authors_text":"Aleksandra Eliseeva, Alexander Bezzubov, Danny Dig, Petr Tsvetkov, Timofey Bryksin, Yaroslav Golubev, Yaroslav Zharov","submitted_at":"2024-10-15T20:32:07Z","abstract_excerpt":"When a Commit Message Generation (CMG) system is integrated into the IDEs and other products at JetBrains, we perform online evaluation based on user acceptance of the generated messages. However, performing online experiments with every change to a CMG system is troublesome, as each iteration affects users and requires time to collect enough statistics. On the other hand, offline evaluation, a prevalent approach in the research literature, facilitates fast experiments but employs automatic metrics that are not guaranteed to represent the preferences of real users. In this work, we describe a "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.12046","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.12046/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2410.12046","created_at":"2026-07-05T09:58:25.373321+00:00"},{"alias_kind":"arxiv_version","alias_value":"2410.12046v2","created_at":"2026-07-05T09:58:25.373321+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.12046","created_at":"2026-07-05T09:58:25.373321+00:00"},{"alias_kind":"pith_short_12","alias_value":"T547BO7OQXXK","created_at":"2026-07-05T09:58:25.373321+00:00"},{"alias_kind":"pith_short_16","alias_value":"T547BO7OQXXKM4QP","created_at":"2026-07-05T09:58:25.373321+00:00"},{"alias_kind":"pith_short_8","alias_value":"T547BO7O","created_at":"2026-07-05T09:58:25.373321+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.02256","citing_title":"CommitSuite: A Comprehensive Benchmark for Commit Classification and Message Generation","ref_index":29,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3","json":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3.json","graph_json":"https://pith.science/api/pith-number/T547BO7OQXXKM4QPCFBGCTUIZ3/graph.json","events_json":"https://pith.science/api/pith-number/T547BO7OQXXKM4QPCFBGCTUIZ3/events.json","paper":"https://pith.science/paper/T547BO7O"},"agent_actions":{"view_html":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3","download_json":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3.json","view_paper":"https://pith.science/paper/T547BO7O","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2410.12046&json=true","fetch_graph":"https://pith.science/api/pith-number/T547BO7OQXXKM4QPCFBGCTUIZ3/graph.json","fetch_events":"https://pith.science/api/pith-number/T547BO7OQXXKM4QPCFBGCTUIZ3/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3/action/timestamp_anchor","attest_storage":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3/action/storage_attestation","attest_author":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3/action/author_attestation","sign_citation":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3/action/citation_signature","submit_replication":"https://pith.science/pith/T547BO7OQXXKM4QPCFBGCTUIZ3/action/replication_record"}},"created_at":"2026-07-05T09:58:25.373321+00:00","updated_at":"2026-07-05T09:58:25.373321+00:00"}