{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:3D2XJQS2ROOCSPFHVJQT6TF44P","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f02e206d7fa188c0e751507a5bf36e6518001e1a801ae3f188da9fd853367633","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-12T13:31:58Z","title_canon_sha256":"d4acf1c617359ed225ad1b81c59f0b00cd1396bbeb81163b02f5aa04ec2ab74c"},"schema_version":"1.0","source":{"id":"2412.09269","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.09269","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"arxiv_version","alias_value":"2412.09269v1","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.09269","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_12","alias_value":"3D2XJQS2ROOC","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_16","alias_value":"3D2XJQS2ROOCSPFH","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_8","alias_value":"3D2XJQS2","created_at":"2026-07-05T09:48:22Z"}],"graph_snapshots":[{"event_id":"sha256:b5ccd2fb041d6f534e5f8dcc305a67cb16bdcdd6d182539b6d376786f859f1cf","target":"graph","created_at":"2026-07-05T09:48:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.09269/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Traditional evaluation metrics like BLEU and ROUGE fall short when capturing the nuanced qualities of generated text, particularly when there is no single ground truth. In this paper, we explore the potential of Large Language Models (LLMs), specifically Google Gemini 1, to serve as automatic evaluators for non-standardized metrics in summarization and dialog-based tasks. We conduct experiments across multiple prompting strategies to examine how LLMs fare as quality evaluators when compared with human judgments on the SummEval and USR datasets, asking the model to generate both a score as well","authors_text":"Harshit Gupta, Manav Chaudhary, Savita Bhat, Vasudeva Varma","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-12T13:31:58Z","title":"Towards Understanding the Robustness of LLM-based Evaluations under Perturbations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.09269","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:01cd3f4552b697cd1b39cb2bb0d5c4e98c61aa6af78cae35d11b7e370c748e4c","target":"record","created_at":"2026-07-05T09:48:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f02e206d7fa188c0e751507a5bf36e6518001e1a801ae3f188da9fd853367633","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-12T13:31:58Z","title_canon_sha256":"d4acf1c617359ed225ad1b81c59f0b00cd1396bbeb81163b02f5aa04ec2ab74c"},"schema_version":"1.0","source":{"id":"2412.09269","kind":"arxiv","version":1}},"canonical_sha256":"d8f574c25a8b9c293ca7aa613f4cbce3c36c4728fe35d3f11566b554e29e9c58","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d8f574c25a8b9c293ca7aa613f4cbce3c36c4728fe35d3f11566b554e29e9c58","first_computed_at":"2026-07-05T09:48:22.192908Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:48:22.192908Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"USbU5aqgmC00RLs/lk1aczWZx042xcSdDaGkCdP7W2f2a2BAR2kkcBqYiQL6iMszzEM8PqY2cPLUDTw/38eoDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:48:22.193400Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.09269","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:01cd3f4552b697cd1b39cb2bb0d5c4e98c61aa6af78cae35d11b7e370c748e4c","sha256:b5ccd2fb041d6f534e5f8dcc305a67cb16bdcdd6d182539b6d376786f859f1cf"],"state_sha256":"b432d488736dae9c535f0e6682b251aa03acd35fff4d067dfe1d1860f313f591"}