{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GVZDDICGV7KGMARPXIOYYHY467","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"01a61cc686e8a91f9aa7aef9eeddb58d068084fcea33e591940a3e34006b31c0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.SE","submitted_at":"2024-10-28T14:34:36Z","title_canon_sha256":"4cf5aa1ae22066fbd7f093cc0018b7f01d834f0f840aed843e747ebd01c6ccc2"},"schema_version":"1.0","source":{"id":"2410.21071","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.21071","created_at":"2026-07-05T09:27:15Z"},{"alias_kind":"arxiv_version","alias_value":"2410.21071v1","created_at":"2026-07-05T09:27:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.21071","created_at":"2026-07-05T09:27:15Z"},{"alias_kind":"pith_short_12","alias_value":"GVZDDICGV7KG","created_at":"2026-07-05T09:27:15Z"},{"alias_kind":"pith_short_16","alias_value":"GVZDDICGV7KGMARP","created_at":"2026-07-05T09:27:15Z"},{"alias_kind":"pith_short_8","alias_value":"GVZDDICG","created_at":"2026-07-05T09:27:15Z"}],"graph_snapshots":[{"event_id":"sha256:e21f16ecd1f6ad27d3b1033046094e69c860daba64ba0724cdf5eac94979fd78","target":"graph","created_at":"2026-07-05T09:27:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.21071/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"LLMs can be used in a variety of code related tasks such as translating from one programming language to another, implementing natural language requirements and code summarization. Artifacts generated by state of the art LLM technology are expected to be useful in the sense that a user will be able to use the LLM generated artifact after a small number of easy modifications. Quantifying this vague notion is challenging and it is thus hard to determine the quality of code related LLM solutions. We refer to evaluation of LLM solutions using LLM judgment as \"LLM as a Judge\", or LaaJ for short. In","authors_text":"Eitan Farchi, Orna Raz, Rami Katan, Shmulik Froimovich","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.SE","submitted_at":"2024-10-28T14:34:36Z","title":"Automatic Generation of Benchmarks and Reliable LLM Judgment for Code Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.21071","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ddf20b34e78bfd8cb6161ff5e2b572175c9a274e1e8d3f174ac5f36e244b868a","target":"record","created_at":"2026-07-05T09:27:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"01a61cc686e8a91f9aa7aef9eeddb58d068084fcea33e591940a3e34006b31c0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.SE","submitted_at":"2024-10-28T14:34:36Z","title_canon_sha256":"4cf5aa1ae22066fbd7f093cc0018b7f01d834f0f840aed843e747ebd01c6ccc2"},"schema_version":"1.0","source":{"id":"2410.21071","kind":"arxiv","version":1}},"canonical_sha256":"357231a046afd466022fba1d8c1f1cf7d130d55d32899c4335fdc7f2b664a20e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"357231a046afd466022fba1d8c1f1cf7d130d55d32899c4335fdc7f2b664a20e","first_computed_at":"2026-07-05T09:27:15.140083Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:27:15.140083Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"WmDWnU7mGGbd8volhhE1/Pw+RCVpp8I/qg+e8hd8ArOGPfOhUGxbNJE0on6/sLwiS7ET9JPgBNl3rxMZ1eT+AQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:27:15.140587Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.21071","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ddf20b34e78bfd8cb6161ff5e2b572175c9a274e1e8d3f174ac5f36e244b868a","sha256:e21f16ecd1f6ad27d3b1033046094e69c860daba64ba0724cdf5eac94979fd78"],"state_sha256":"2c233e27aba169eba1af6dacbe783afffe5ed5a533ebb702bed2c20a9cf01888"}