{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:Q4PZPKSFST3VZIA6ATRXZYLHID","short_pith_number":"pith:Q4PZPKSF","canonical_record":{"source":{"id":"2508.02827","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-08-04T18:52:01Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ef23df3f0665ac4a0f8c369eb7c210394b77245ad19ca24d96740c89886e58db","abstract_canon_sha256":"de1ed1edd4227616d74be1f424d7337727ec83484ea9914ff3b124ef3c600426"},"schema_version":"1.0"},"canonical_sha256":"871f97aa4594f75ca01e04e37ce16740f610613cae4b90fbcf9cf1e743ab48fa","source":{"kind":"arxiv","id":"2508.02827","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2508.02827","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"arxiv_version","alias_value":"2508.02827v1","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.02827","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"pith_short_12","alias_value":"Q4PZPKSFST3V","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"pith_short_16","alias_value":"Q4PZPKSFST3VZIA6","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"pith_short_8","alias_value":"Q4PZPKSF","created_at":"2026-07-05T11:48:31Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:Q4PZPKSFST3VZIA6ATRXZYLHID","target":"record","payload":{"canonical_record":{"source":{"id":"2508.02827","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-08-04T18:52:01Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ef23df3f0665ac4a0f8c369eb7c210394b77245ad19ca24d96740c89886e58db","abstract_canon_sha256":"de1ed1edd4227616d74be1f424d7337727ec83484ea9914ff3b124ef3c600426"},"schema_version":"1.0"},"canonical_sha256":"871f97aa4594f75ca01e04e37ce16740f610613cae4b90fbcf9cf1e743ab48fa","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:48:31.698531Z","signature_b64":"bHfmwrP0HUU0d8LKqkoSZEXSAoTfWQsPjvFQ3b/dZZNOC7cZDVedEO03QkgoCDp5wthHyXbYYPjnK8j6uhABDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"871f97aa4594f75ca01e04e37ce16740f610613cae4b90fbcf9cf1e743ab48fa","last_reissued_at":"2026-07-05T11:48:31.698010Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:48:31.698010Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2508.02827","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:48:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"C93wvX7QPTD9ahkvT7IWXE7O4zt9lVVxLPnOAxtcThQH7R4wowPobKCDtpiswvt9u0rjoId6hFqMJ6lgY4PDDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T22:41:52.389266Z"},"content_sha256":"47535e8ca950cfbd6587b1a11ddbb8c992ac98b3170aaed97ec360f304e982d0","schema_version":"1.0","event_id":"sha256:47535e8ca950cfbd6587b1a11ddbb8c992ac98b3170aaed97ec360f304e982d0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:Q4PZPKSFST3VZIA6ATRXZYLHID","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Automated Validation of LLM-based Evaluators for Software Engineering Artifacts","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.SE","authors_text":"Alice Podolsky, Avi Ziv, Eitan Farchi, Ora Nova Fandina, Orna Raz, Rami Katan, Shmulik Froimovich","submitted_at":"2025-08-04T18:52:01Z","abstract_excerpt":"Automation in software engineering increasingly relies on large language models (LLMs) to generate, review, and assess code artifacts. However, establishing LLMs as reliable evaluators remains an open challenge: human evaluations are costly, subjective and non scalable, while existing automated methods fail to discern fine grained variations in artifact quality.\n  We introduce REFINE (Ranking Evaluators for FIne grained Nuanced Evaluation), an automated framework for benchmarking LLM based evaluators across software engineering tasks. REFINE comprises of two modules: Hierarchy Dataset Builder "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.02827","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2508.02827/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:48:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"b0c7HhWxM08vclGrS2fu25ns0udcBuJy/EBuU+pS88R9VSQloTsHtXTKnI0bd4nlrh+YctBEq74YOqIRZOpqAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T22:41:52.389777Z"},"content_sha256":"6bb1208ac10bc4545051aabc0403c88ce7bb8cc0896eaf476ddd82464905e7ea","schema_version":"1.0","event_id":"sha256:6bb1208ac10bc4545051aabc0403c88ce7bb8cc0896eaf476ddd82464905e7ea"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Q4PZPKSFST3VZIA6ATRXZYLHID/bundle.json","state_url":"https://pith.science/pith/Q4PZPKSFST3VZIA6ATRXZYLHID/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Q4PZPKSFST3VZIA6ATRXZYLHID/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T22:41:52Z","links":{"resolver":"https://pith.science/pith/Q4PZPKSFST3VZIA6ATRXZYLHID","bundle":"https://pith.science/pith/Q4PZPKSFST3VZIA6ATRXZYLHID/bundle.json","state":"https://pith.science/pith/Q4PZPKSFST3VZIA6ATRXZYLHID/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Q4PZPKSFST3VZIA6ATRXZYLHID/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:Q4PZPKSFST3VZIA6ATRXZYLHID","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"de1ed1edd4227616d74be1f424d7337727ec83484ea9914ff3b124ef3c600426","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-08-04T18:52:01Z","title_canon_sha256":"ef23df3f0665ac4a0f8c369eb7c210394b77245ad19ca24d96740c89886e58db"},"schema_version":"1.0","source":{"id":"2508.02827","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2508.02827","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"arxiv_version","alias_value":"2508.02827v1","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.02827","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"pith_short_12","alias_value":"Q4PZPKSFST3V","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"pith_short_16","alias_value":"Q4PZPKSFST3VZIA6","created_at":"2026-07-05T11:48:31Z"},{"alias_kind":"pith_short_8","alias_value":"Q4PZPKSF","created_at":"2026-07-05T11:48:31Z"}],"graph_snapshots":[{"event_id":"sha256:6bb1208ac10bc4545051aabc0403c88ce7bb8cc0896eaf476ddd82464905e7ea","target":"graph","created_at":"2026-07-05T11:48:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2508.02827/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Automation in software engineering increasingly relies on large language models (LLMs) to generate, review, and assess code artifacts. However, establishing LLMs as reliable evaluators remains an open challenge: human evaluations are costly, subjective and non scalable, while existing automated methods fail to discern fine grained variations in artifact quality.\n  We introduce REFINE (Ranking Evaluators for FIne grained Nuanced Evaluation), an automated framework for benchmarking LLM based evaluators across software engineering tasks. REFINE comprises of two modules: Hierarchy Dataset Builder ","authors_text":"Alice Podolsky, Avi Ziv, Eitan Farchi, Ora Nova Fandina, Orna Raz, Rami Katan, Shmulik Froimovich","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-08-04T18:52:01Z","title":"Automated Validation of LLM-based Evaluators for Software Engineering Artifacts"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.02827","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:47535e8ca950cfbd6587b1a11ddbb8c992ac98b3170aaed97ec360f304e982d0","target":"record","created_at":"2026-07-05T11:48:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"de1ed1edd4227616d74be1f424d7337727ec83484ea9914ff3b124ef3c600426","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-08-04T18:52:01Z","title_canon_sha256":"ef23df3f0665ac4a0f8c369eb7c210394b77245ad19ca24d96740c89886e58db"},"schema_version":"1.0","source":{"id":"2508.02827","kind":"arxiv","version":1}},"canonical_sha256":"871f97aa4594f75ca01e04e37ce16740f610613cae4b90fbcf9cf1e743ab48fa","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"871f97aa4594f75ca01e04e37ce16740f610613cae4b90fbcf9cf1e743ab48fa","first_computed_at":"2026-07-05T11:48:31.698010Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:48:31.698010Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"bHfmwrP0HUU0d8LKqkoSZEXSAoTfWQsPjvFQ3b/dZZNOC7cZDVedEO03QkgoCDp5wthHyXbYYPjnK8j6uhABDw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:48:31.698531Z","signed_message":"canonical_sha256_bytes"},"source_id":"2508.02827","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:47535e8ca950cfbd6587b1a11ddbb8c992ac98b3170aaed97ec360f304e982d0","sha256:6bb1208ac10bc4545051aabc0403c88ce7bb8cc0896eaf476ddd82464905e7ea"],"state_sha256":"32e0998e3bddfc097a3fc568a32bdb38966cfec91ea3586afc1cc79dfcf921a4"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rgs/gc1JfroKPTyA8kD7skuF/X9ke6SdoMUqbgJPOrDOCbGeTLjY6lT6ipiQbvJtufKkUMsWSBxHaosUBX1HCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T22:41:52.393505Z","bundle_sha256":"e33cbeeef61fe2b5de75362ef64a9c5dc99bcf2d6c0b1be5dbd7f68e7b59bed5"}}