{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:3D2XJQS2ROOCSPFHVJQT6TF44P","short_pith_number":"pith:3D2XJQS2","canonical_record":{"source":{"id":"2412.09269","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-12T13:31:58Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"d4acf1c617359ed225ad1b81c59f0b00cd1396bbeb81163b02f5aa04ec2ab74c","abstract_canon_sha256":"f02e206d7fa188c0e751507a5bf36e6518001e1a801ae3f188da9fd853367633"},"schema_version":"1.0"},"canonical_sha256":"d8f574c25a8b9c293ca7aa613f4cbce3c36c4728fe35d3f11566b554e29e9c58","source":{"kind":"arxiv","id":"2412.09269","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.09269","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"arxiv_version","alias_value":"2412.09269v1","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.09269","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_12","alias_value":"3D2XJQS2ROOC","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_16","alias_value":"3D2XJQS2ROOCSPFH","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_8","alias_value":"3D2XJQS2","created_at":"2026-07-05T09:48:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:3D2XJQS2ROOCSPFHVJQT6TF44P","target":"record","payload":{"canonical_record":{"source":{"id":"2412.09269","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-12T13:31:58Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"d4acf1c617359ed225ad1b81c59f0b00cd1396bbeb81163b02f5aa04ec2ab74c","abstract_canon_sha256":"f02e206d7fa188c0e751507a5bf36e6518001e1a801ae3f188da9fd853367633"},"schema_version":"1.0"},"canonical_sha256":"d8f574c25a8b9c293ca7aa613f4cbce3c36c4728fe35d3f11566b554e29e9c58","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:48:22.193400Z","signature_b64":"USbU5aqgmC00RLs/lk1aczWZx042xcSdDaGkCdP7W2f2a2BAR2kkcBqYiQL6iMszzEM8PqY2cPLUDTw/38eoDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d8f574c25a8b9c293ca7aa613f4cbce3c36c4728fe35d3f11566b554e29e9c58","last_reissued_at":"2026-07-05T09:48:22.192908Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:48:22.192908Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2412.09269","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:48:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"7gHCXJE/0R7N+IImkMNUkz0dkhpAiSi3dg8IYnUNDNinQhETRSj8egvUzLadMNZM8lKpJ/8LcS8qsGgHTXD7Bw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-28T02:40:13.989841Z"},"content_sha256":"01cd3f4552b697cd1b39cb2bb0d5c4e98c61aa6af78cae35d11b7e370c748e4c","schema_version":"1.0","event_id":"sha256:01cd3f4552b697cd1b39cb2bb0d5c4e98c61aa6af78cae35d11b7e370c748e4c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:3D2XJQS2ROOCSPFHVJQT6TF44P","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Towards Understanding the Robustness of LLM-based Evaluations under Perturbations","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Harshit Gupta, Manav Chaudhary, Savita Bhat, Vasudeva Varma","submitted_at":"2024-12-12T13:31:58Z","abstract_excerpt":"Traditional evaluation metrics like BLEU and ROUGE fall short when capturing the nuanced qualities of generated text, particularly when there is no single ground truth. In this paper, we explore the potential of Large Language Models (LLMs), specifically Google Gemini 1, to serve as automatic evaluators for non-standardized metrics in summarization and dialog-based tasks. We conduct experiments across multiple prompting strategies to examine how LLMs fare as quality evaluators when compared with human judgments on the SummEval and USR datasets, asking the model to generate both a score as well"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.09269","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.09269/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:48:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jeuXHIcq8PcyXxDgD5W+H0fSis1kwgmsuNxNupWGAJ9wGPzlkip1TI8U+S3fQueixy1FZMtPPs1SesN0nlOgCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-28T02:40:13.990211Z"},"content_sha256":"b5ccd2fb041d6f534e5f8dcc305a67cb16bdcdd6d182539b6d376786f859f1cf","schema_version":"1.0","event_id":"sha256:b5ccd2fb041d6f534e5f8dcc305a67cb16bdcdd6d182539b6d376786f859f1cf"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3D2XJQS2ROOCSPFHVJQT6TF44P/bundle.json","state_url":"https://pith.science/pith/3D2XJQS2ROOCSPFHVJQT6TF44P/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3D2XJQS2ROOCSPFHVJQT6TF44P/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-28T02:40:13Z","links":{"resolver":"https://pith.science/pith/3D2XJQS2ROOCSPFHVJQT6TF44P","bundle":"https://pith.science/pith/3D2XJQS2ROOCSPFHVJQT6TF44P/bundle.json","state":"https://pith.science/pith/3D2XJQS2ROOCSPFHVJQT6TF44P/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3D2XJQS2ROOCSPFHVJQT6TF44P/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:3D2XJQS2ROOCSPFHVJQT6TF44P","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f02e206d7fa188c0e751507a5bf36e6518001e1a801ae3f188da9fd853367633","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-12T13:31:58Z","title_canon_sha256":"d4acf1c617359ed225ad1b81c59f0b00cd1396bbeb81163b02f5aa04ec2ab74c"},"schema_version":"1.0","source":{"id":"2412.09269","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.09269","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"arxiv_version","alias_value":"2412.09269v1","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.09269","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_12","alias_value":"3D2XJQS2ROOC","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_16","alias_value":"3D2XJQS2ROOCSPFH","created_at":"2026-07-05T09:48:22Z"},{"alias_kind":"pith_short_8","alias_value":"3D2XJQS2","created_at":"2026-07-05T09:48:22Z"}],"graph_snapshots":[{"event_id":"sha256:b5ccd2fb041d6f534e5f8dcc305a67cb16bdcdd6d182539b6d376786f859f1cf","target":"graph","created_at":"2026-07-05T09:48:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.09269/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Traditional evaluation metrics like BLEU and ROUGE fall short when capturing the nuanced qualities of generated text, particularly when there is no single ground truth. In this paper, we explore the potential of Large Language Models (LLMs), specifically Google Gemini 1, to serve as automatic evaluators for non-standardized metrics in summarization and dialog-based tasks. We conduct experiments across multiple prompting strategies to examine how LLMs fare as quality evaluators when compared with human judgments on the SummEval and USR datasets, asking the model to generate both a score as well","authors_text":"Harshit Gupta, Manav Chaudhary, Savita Bhat, Vasudeva Varma","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-12T13:31:58Z","title":"Towards Understanding the Robustness of LLM-based Evaluations under Perturbations"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.09269","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:01cd3f4552b697cd1b39cb2bb0d5c4e98c61aa6af78cae35d11b7e370c748e4c","target":"record","created_at":"2026-07-05T09:48:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f02e206d7fa188c0e751507a5bf36e6518001e1a801ae3f188da9fd853367633","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-12-12T13:31:58Z","title_canon_sha256":"d4acf1c617359ed225ad1b81c59f0b00cd1396bbeb81163b02f5aa04ec2ab74c"},"schema_version":"1.0","source":{"id":"2412.09269","kind":"arxiv","version":1}},"canonical_sha256":"d8f574c25a8b9c293ca7aa613f4cbce3c36c4728fe35d3f11566b554e29e9c58","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"d8f574c25a8b9c293ca7aa613f4cbce3c36c4728fe35d3f11566b554e29e9c58","first_computed_at":"2026-07-05T09:48:22.192908Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:48:22.192908Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"USbU5aqgmC00RLs/lk1aczWZx042xcSdDaGkCdP7W2f2a2BAR2kkcBqYiQL6iMszzEM8PqY2cPLUDTw/38eoDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:48:22.193400Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.09269","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:01cd3f4552b697cd1b39cb2bb0d5c4e98c61aa6af78cae35d11b7e370c748e4c","sha256:b5ccd2fb041d6f534e5f8dcc305a67cb16bdcdd6d182539b6d376786f859f1cf"],"state_sha256":"b432d488736dae9c535f0e6682b251aa03acd35fff4d067dfe1d1860f313f591"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0gtmuGxHRt4N71N+RirIymvNyh7Pwq4FF7oc6p6OhrRUy3EmsxPal91bBTdhaJA7Vbl5KuVVFIuPXevjaRweDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-28T02:40:13.992632Z","bundle_sha256":"6ebe225f16d0f7d752c49c5d975f8f2d0addb8af38e453824999586012b21bc0"}}