{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:L62TIQMYK2PKS4MWX3LXDM553H","short_pith_number":"pith:L62TIQMY","canonical_record":{"source":{"id":"2501.14883","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-24T19:17:06Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"2a095dad87a7813e6b97e90882456eb6944f4f1dc7eac447dd75e72b52311d61","abstract_canon_sha256":"bd277cf84640ba7b9ae0f610be10d6094f4821adc3cdbcfd667ed0915812b761"},"schema_version":"1.0"},"canonical_sha256":"5fb5344198569ea97196bed771b3bdd9ded52111dffd5bc13b94ce472bddbbe9","source":{"kind":"arxiv","id":"2501.14883","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.14883","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"arxiv_version","alias_value":"2501.14883v2","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.14883","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"pith_short_12","alias_value":"L62TIQMYK2PK","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"pith_short_16","alias_value":"L62TIQMYK2PKS4MW","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"pith_short_8","alias_value":"L62TIQMY","created_at":"2026-07-05T10:07:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:L62TIQMYK2PKS4MWX3LXDM553H","target":"record","payload":{"canonical_record":{"source":{"id":"2501.14883","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-24T19:17:06Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"2a095dad87a7813e6b97e90882456eb6944f4f1dc7eac447dd75e72b52311d61","abstract_canon_sha256":"bd277cf84640ba7b9ae0f610be10d6094f4821adc3cdbcfd667ed0915812b761"},"schema_version":"1.0"},"canonical_sha256":"5fb5344198569ea97196bed771b3bdd9ded52111dffd5bc13b94ce472bddbbe9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:07:29.718632Z","signature_b64":"1I+um26YFkhRjJNOX6Q+piAQf6DAoDzNf6498ZAmIvSOY8CDHPHyImzb2rd4qmwnTd+SEQV/e/MoQPot04azDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5fb5344198569ea97196bed771b3bdd9ded52111dffd5bc13b94ce472bddbbe9","last_reissued_at":"2026-07-05T10:07:29.718071Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:07:29.718071Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2501.14883","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:07:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5ovRDPQOlD8lqdZBApLokHwDrLbHfnNmAfcKmSJknJk+nY5sH0pU8pOaROJugimWhVGg682B8ABzmqwTiP91Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T15:01:04.051366Z"},"content_sha256":"280f1d0feef6dfb669f2ee43ef7d4625986793a9600c3398a003b1a14a4812d0","schema_version":"1.0","event_id":"sha256:280f1d0feef6dfb669f2ee43ef7d4625986793a9600c3398a003b1a14a4812d0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:L62TIQMYK2PKS4MWX3LXDM553H","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Verify with Caution: The Pitfalls of Relying on Imperfect Factuality Metrics","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Ameya Godbole, Robin Jia","submitted_at":"2025-01-24T19:17:06Z","abstract_excerpt":"Improvements in large language models have led to increasing optimism that they can serve as reliable evaluators of natural language generation outputs. In this paper, we challenge this optimism by thoroughly re-evaluating five state-of-the-art factuality metrics on a collection of 11 datasets for summarization, retrieval-augmented generation, and question answering. We find that these evaluators are inconsistent with each other and often misestimate system-level performance, both of which can lead to a variety of pitfalls. We further show that these metrics exhibit biases against highly parap"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.14883","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.14883/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:07:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"IMtQPlHhhHaASlpAqc/k6bJc8mDnxn2KDDsFq9Ch3JsoLmZ5069JmxffmE7mzZsTugTrb8TaF3r/n0snpQgGAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T15:01:04.052106Z"},"content_sha256":"eac3c2b638836c349b7cab6287ad0ba48cbc8dc940d27a7cd9228f6ab6c91de7","schema_version":"1.0","event_id":"sha256:eac3c2b638836c349b7cab6287ad0ba48cbc8dc940d27a7cd9228f6ab6c91de7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/L62TIQMYK2PKS4MWX3LXDM553H/bundle.json","state_url":"https://pith.science/pith/L62TIQMYK2PKS4MWX3LXDM553H/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/L62TIQMYK2PKS4MWX3LXDM553H/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T15:01:04Z","links":{"resolver":"https://pith.science/pith/L62TIQMYK2PKS4MWX3LXDM553H","bundle":"https://pith.science/pith/L62TIQMYK2PKS4MWX3LXDM553H/bundle.json","state":"https://pith.science/pith/L62TIQMYK2PKS4MWX3LXDM553H/state.json","well_known_bundle":"https://pith.science/.well-known/pith/L62TIQMYK2PKS4MWX3LXDM553H/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:L62TIQMYK2PKS4MWX3LXDM553H","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bd277cf84640ba7b9ae0f610be10d6094f4821adc3cdbcfd667ed0915812b761","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-24T19:17:06Z","title_canon_sha256":"2a095dad87a7813e6b97e90882456eb6944f4f1dc7eac447dd75e72b52311d61"},"schema_version":"1.0","source":{"id":"2501.14883","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2501.14883","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"arxiv_version","alias_value":"2501.14883v2","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.14883","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"pith_short_12","alias_value":"L62TIQMYK2PK","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"pith_short_16","alias_value":"L62TIQMYK2PKS4MW","created_at":"2026-07-05T10:07:29Z"},{"alias_kind":"pith_short_8","alias_value":"L62TIQMY","created_at":"2026-07-05T10:07:29Z"}],"graph_snapshots":[{"event_id":"sha256:eac3c2b638836c349b7cab6287ad0ba48cbc8dc940d27a7cd9228f6ab6c91de7","target":"graph","created_at":"2026-07-05T10:07:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2501.14883/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Improvements in large language models have led to increasing optimism that they can serve as reliable evaluators of natural language generation outputs. In this paper, we challenge this optimism by thoroughly re-evaluating five state-of-the-art factuality metrics on a collection of 11 datasets for summarization, retrieval-augmented generation, and question answering. We find that these evaluators are inconsistent with each other and often misestimate system-level performance, both of which can lead to a variety of pitfalls. We further show that these metrics exhibit biases against highly parap","authors_text":"Ameya Godbole, Robin Jia","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-24T19:17:06Z","title":"Verify with Caution: The Pitfalls of Relying on Imperfect Factuality Metrics"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.14883","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:280f1d0feef6dfb669f2ee43ef7d4625986793a9600c3398a003b1a14a4812d0","target":"record","created_at":"2026-07-05T10:07:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bd277cf84640ba7b9ae0f610be10d6094f4821adc3cdbcfd667ed0915812b761","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-24T19:17:06Z","title_canon_sha256":"2a095dad87a7813e6b97e90882456eb6944f4f1dc7eac447dd75e72b52311d61"},"schema_version":"1.0","source":{"id":"2501.14883","kind":"arxiv","version":2}},"canonical_sha256":"5fb5344198569ea97196bed771b3bdd9ded52111dffd5bc13b94ce472bddbbe9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5fb5344198569ea97196bed771b3bdd9ded52111dffd5bc13b94ce472bddbbe9","first_computed_at":"2026-07-05T10:07:29.718071Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:07:29.718071Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"1I+um26YFkhRjJNOX6Q+piAQf6DAoDzNf6498ZAmIvSOY8CDHPHyImzb2rd4qmwnTd+SEQV/e/MoQPot04azDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:07:29.718632Z","signed_message":"canonical_sha256_bytes"},"source_id":"2501.14883","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:280f1d0feef6dfb669f2ee43ef7d4625986793a9600c3398a003b1a14a4812d0","sha256:eac3c2b638836c349b7cab6287ad0ba48cbc8dc940d27a7cd9228f6ab6c91de7"],"state_sha256":"2352572ced57034280da457fdf4f35194b226c09d12c34ac7c15346f6bd12a36"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aEyk2N99Ok8AOclCLPoT5+mXD/J7YdcHROdSBoXRxRx2szcmEPf3a9PimIRN5i0/fCftCB4SB3D6DTV2pzm+DA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T15:01:04.060530Z","bundle_sha256":"17b257228d3ae509de2ed1b503fe592c179cbb15c5b9415f181994302afeb67d"}}