{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:XEGI62BJDXCS4IN27UTKZVHY3L","short_pith_number":"pith:XEGI62BJ","canonical_record":{"source":{"id":"2402.11161","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-17T01:56:19Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"aa9e3987242e98b679da36327755e268159ef09e3c9d58c38b4f84b3bbe755ac","abstract_canon_sha256":"b074fbca825ad626ca2b90267e3af47871a0b454dd2eb5d247adceee551d7996"},"schema_version":"1.0"},"canonical_sha256":"b90c8f68291dc52e21bafd26acd4f8dad900d837e2cb4e847e079bb8344e133d","source":{"kind":"arxiv","id":"2402.11161","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.11161","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"arxiv_version","alias_value":"2402.11161v5","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.11161","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"pith_short_12","alias_value":"XEGI62BJDXCS","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"pith_short_16","alias_value":"XEGI62BJDXCS4IN2","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"pith_short_8","alias_value":"XEGI62BJ","created_at":"2026-07-05T09:19:26Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:XEGI62BJDXCS4IN27UTKZVHY3L","target":"record","payload":{"canonical_record":{"source":{"id":"2402.11161","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-17T01:56:19Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"aa9e3987242e98b679da36327755e268159ef09e3c9d58c38b4f84b3bbe755ac","abstract_canon_sha256":"b074fbca825ad626ca2b90267e3af47871a0b454dd2eb5d247adceee551d7996"},"schema_version":"1.0"},"canonical_sha256":"b90c8f68291dc52e21bafd26acd4f8dad900d837e2cb4e847e079bb8344e133d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:19:26.505257Z","signature_b64":"GpnIq+t6g4Y+JtLT9sFJjmbxMVLO2/nxkgQIni55H2OCeJTqAqCkhTdZn31rn4jsmp7rHscOD+osUd8six4GCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b90c8f68291dc52e21bafd26acd4f8dad900d837e2cb4e847e079bb8344e133d","last_reissued_at":"2026-07-05T09:19:26.504753Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:19:26.504753Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.11161","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:19:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Ot3p3hBIphB2a89pKa6B7kXDrOwf7w9FAuy8MGnt9J3OcMihzrk9aUKc+lWzlS/QjMHDLm98ExwZZ4+/EMCBBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T17:46:58.037141Z"},"content_sha256":"f2a0c7e8d3eb430d99f3c0a2e506829f326c4ab55026c2f9beefd99b90eb5311","schema_version":"1.0","event_id":"sha256:f2a0c7e8d3eb430d99f3c0a2e506829f326c4ab55026c2f9beefd99b90eb5311"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:XEGI62BJDXCS4IN27UTKZVHY3L","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"PEDANTS: Cheap but Effective and Interpretable Answer Equivalence","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Huy Nghiem, Ishani Mondal, Jordan Lee Boyd-Graber, Yijun Liang, Zongxia Li","submitted_at":"2024-02-17T01:56:19Z","abstract_excerpt":"Question answering (QA) can only make progress if we know if an answer is correct, but current answer correctness (AC) metrics struggle with verbose, free-form answers from large language models (LLMs). There are two challenges with current short-form QA evaluations: a lack of diverse styles of evaluation data and an over-reliance on expensive and slow LLMs. LLM-based scorers correlate better with humans, but this expensive task has only been tested on limited QA datasets. We rectify these issues by providing rubrics and datasets for evaluating machine QA adopted from the Trivia community. We "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.11161","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.11161/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:19:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"fcpE8ixqGcAYXx81s6JsBH5xqTbsGIPpvH4pA8KocfbPSOv+57I9symvyT6LbN5HwNwplSVtmj14o9Cc07YmAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-16T17:46:58.037732Z"},"content_sha256":"e1611ed41578449821961d95896ecc78549a0e3a340fe97a8030f70c83caa6c3","schema_version":"1.0","event_id":"sha256:e1611ed41578449821961d95896ecc78549a0e3a340fe97a8030f70c83caa6c3"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/XEGI62BJDXCS4IN27UTKZVHY3L/bundle.json","state_url":"https://pith.science/pith/XEGI62BJDXCS4IN27UTKZVHY3L/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/XEGI62BJDXCS4IN27UTKZVHY3L/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-16T17:46:58Z","links":{"resolver":"https://pith.science/pith/XEGI62BJDXCS4IN27UTKZVHY3L","bundle":"https://pith.science/pith/XEGI62BJDXCS4IN27UTKZVHY3L/bundle.json","state":"https://pith.science/pith/XEGI62BJDXCS4IN27UTKZVHY3L/state.json","well_known_bundle":"https://pith.science/.well-known/pith/XEGI62BJDXCS4IN27UTKZVHY3L/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:XEGI62BJDXCS4IN27UTKZVHY3L","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"b074fbca825ad626ca2b90267e3af47871a0b454dd2eb5d247adceee551d7996","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-17T01:56:19Z","title_canon_sha256":"aa9e3987242e98b679da36327755e268159ef09e3c9d58c38b4f84b3bbe755ac"},"schema_version":"1.0","source":{"id":"2402.11161","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.11161","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"arxiv_version","alias_value":"2402.11161v5","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.11161","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"pith_short_12","alias_value":"XEGI62BJDXCS","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"pith_short_16","alias_value":"XEGI62BJDXCS4IN2","created_at":"2026-07-05T09:19:26Z"},{"alias_kind":"pith_short_8","alias_value":"XEGI62BJ","created_at":"2026-07-05T09:19:26Z"}],"graph_snapshots":[{"event_id":"sha256:e1611ed41578449821961d95896ecc78549a0e3a340fe97a8030f70c83caa6c3","target":"graph","created_at":"2026-07-05T09:19:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.11161/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Question answering (QA) can only make progress if we know if an answer is correct, but current answer correctness (AC) metrics struggle with verbose, free-form answers from large language models (LLMs). There are two challenges with current short-form QA evaluations: a lack of diverse styles of evaluation data and an over-reliance on expensive and slow LLMs. LLM-based scorers correlate better with humans, but this expensive task has only been tested on limited QA datasets. We rectify these issues by providing rubrics and datasets for evaluating machine QA adopted from the Trivia community. We ","authors_text":"Huy Nghiem, Ishani Mondal, Jordan Lee Boyd-Graber, Yijun Liang, Zongxia Li","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-17T01:56:19Z","title":"PEDANTS: Cheap but Effective and Interpretable Answer Equivalence"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.11161","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f2a0c7e8d3eb430d99f3c0a2e506829f326c4ab55026c2f9beefd99b90eb5311","target":"record","created_at":"2026-07-05T09:19:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"b074fbca825ad626ca2b90267e3af47871a0b454dd2eb5d247adceee551d7996","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-02-17T01:56:19Z","title_canon_sha256":"aa9e3987242e98b679da36327755e268159ef09e3c9d58c38b4f84b3bbe755ac"},"schema_version":"1.0","source":{"id":"2402.11161","kind":"arxiv","version":5}},"canonical_sha256":"b90c8f68291dc52e21bafd26acd4f8dad900d837e2cb4e847e079bb8344e133d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"b90c8f68291dc52e21bafd26acd4f8dad900d837e2cb4e847e079bb8344e133d","first_computed_at":"2026-07-05T09:19:26.504753Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:19:26.504753Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"GpnIq+t6g4Y+JtLT9sFJjmbxMVLO2/nxkgQIni55H2OCeJTqAqCkhTdZn31rn4jsmp7rHscOD+osUd8six4GCA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:19:26.505257Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.11161","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f2a0c7e8d3eb430d99f3c0a2e506829f326c4ab55026c2f9beefd99b90eb5311","sha256:e1611ed41578449821961d95896ecc78549a0e3a340fe97a8030f70c83caa6c3"],"state_sha256":"1103ac3bcae2cb277dde86fb84cbf14f02764de6209eb9e407ff1d286cf4c452"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wWu6YndOgCFd4RQDKjjxba7Cv/0ikmqqKvqDYHSxtJliA4fjrOXHiuquFGjuVAc+4t4EOrlgLwPAIFsfOiUuDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-16T17:46:58.041704Z","bundle_sha256":"04aa10b56ec4e68a9eccbca7dd80042d60bc597cd1765fc5c2142f9e86369ca1"}}