{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:B4EDHO6P6JJTEFBSZMGPI7BROL","short_pith_number":"pith:B4EDHO6P","canonical_record":{"source":{"id":"2502.03711","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-06T01:58:48Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"edb6c7540631727f68bd37266a135b94364f0ee75ae1de87ba534fa8fb382b19","abstract_canon_sha256":"db1c8ef3ffb893c4585efd88007a9a82d43fd9cd09aa61715e0fa969289e34bb"},"schema_version":"1.0"},"canonical_sha256":"0f0833bbcff253321432cb0cf47c3172e51c52d2578ea67ff5110dfb0b4fdfe0","source":{"kind":"arxiv","id":"2502.03711","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.03711","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"arxiv_version","alias_value":"2502.03711v1","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.03711","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"pith_short_12","alias_value":"B4EDHO6P6JJT","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"pith_short_16","alias_value":"B4EDHO6P6JJTEFBS","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"pith_short_8","alias_value":"B4EDHO6P","created_at":"2026-07-05T10:10:14Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:B4EDHO6P6JJTEFBSZMGPI7BROL","target":"record","payload":{"canonical_record":{"source":{"id":"2502.03711","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-06T01:58:48Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"edb6c7540631727f68bd37266a135b94364f0ee75ae1de87ba534fa8fb382b19","abstract_canon_sha256":"db1c8ef3ffb893c4585efd88007a9a82d43fd9cd09aa61715e0fa969289e34bb"},"schema_version":"1.0"},"canonical_sha256":"0f0833bbcff253321432cb0cf47c3172e51c52d2578ea67ff5110dfb0b4fdfe0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:10:14.037001Z","signature_b64":"6mBSzKVctsktWYQc+xY1liMUgfvrgS0XlsK/+AiALi9jUbdrLq4Akwxxws3qdnbHPwtJWZbl5tntg5qduKpDAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0f0833bbcff253321432cb0cf47c3172e51c52d2578ea67ff5110dfb0b4fdfe0","last_reissued_at":"2026-07-05T10:10:14.036589Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:10:14.036589Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.03711","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:10:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"LQKg+c0RYCkwH0XBpqgguLi4+hZytXGtPs6J4pXxlXATVIvycB3M/mXAaYq+bWDuxlMrzSa4BM8rzCEcLacNBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-25T21:37:44.499579Z"},"content_sha256":"6d4c70f23f096ddd3151b3f138aeb15fda57184738c4adaa405070e38a365da3","schema_version":"1.0","event_id":"sha256:6d4c70f23f096ddd3151b3f138aeb15fda57184738c4adaa405070e38a365da3"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:B4EDHO6P6JJTEFBSZMGPI7BROL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"MultiQ&A: An Analysis in Measuring Robustness via Automated Crowdsourcing of Question Perturbations and Answers","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Nicole Cho, William Watson","submitted_at":"2025-02-06T01:58:48Z","abstract_excerpt":"One critical challenge in the institutional adoption journey of Large Language Models (LLMs) stems from their propensity to hallucinate in generated responses. To address this, we propose MultiQ&A, a systematic approach for evaluating the robustness and consistency of LLM-generated answers. We demonstrate MultiQ&A's ability to crowdsource question perturbations and their respective answers through independent LLM agents at scale. Our experiments culminated in the examination of 1.9 million question perturbations and 2.3 million answers. Furthermore, MultiQ&A shows that ensembled LLMs, such as "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.03711","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.03711/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:10:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JrN+OI1w5xs6Yft2ZqzpMTCyjmX5gkVYYOEW6dpnsXILmAGpgHYT+cMQZJflROIkdSCSyF1llFFgOf/mE0F6DA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-25T21:37:44.499975Z"},"content_sha256":"e54dd0fffd6b26edaa4dfb76ae3597b8157be3d5cb3fdd8ed02643015b5b98d7","schema_version":"1.0","event_id":"sha256:e54dd0fffd6b26edaa4dfb76ae3597b8157be3d5cb3fdd8ed02643015b5b98d7"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/B4EDHO6P6JJTEFBSZMGPI7BROL/bundle.json","state_url":"https://pith.science/pith/B4EDHO6P6JJTEFBSZMGPI7BROL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/B4EDHO6P6JJTEFBSZMGPI7BROL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-25T21:37:44Z","links":{"resolver":"https://pith.science/pith/B4EDHO6P6JJTEFBSZMGPI7BROL","bundle":"https://pith.science/pith/B4EDHO6P6JJTEFBSZMGPI7BROL/bundle.json","state":"https://pith.science/pith/B4EDHO6P6JJTEFBSZMGPI7BROL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/B4EDHO6P6JJTEFBSZMGPI7BROL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:B4EDHO6P6JJTEFBSZMGPI7BROL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"db1c8ef3ffb893c4585efd88007a9a82d43fd9cd09aa61715e0fa969289e34bb","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-06T01:58:48Z","title_canon_sha256":"edb6c7540631727f68bd37266a135b94364f0ee75ae1de87ba534fa8fb382b19"},"schema_version":"1.0","source":{"id":"2502.03711","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.03711","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"arxiv_version","alias_value":"2502.03711v1","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.03711","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"pith_short_12","alias_value":"B4EDHO6P6JJT","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"pith_short_16","alias_value":"B4EDHO6P6JJTEFBS","created_at":"2026-07-05T10:10:14Z"},{"alias_kind":"pith_short_8","alias_value":"B4EDHO6P","created_at":"2026-07-05T10:10:14Z"}],"graph_snapshots":[{"event_id":"sha256:e54dd0fffd6b26edaa4dfb76ae3597b8157be3d5cb3fdd8ed02643015b5b98d7","target":"graph","created_at":"2026-07-05T10:10:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.03711/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"One critical challenge in the institutional adoption journey of Large Language Models (LLMs) stems from their propensity to hallucinate in generated responses. To address this, we propose MultiQ&A, a systematic approach for evaluating the robustness and consistency of LLM-generated answers. We demonstrate MultiQ&A's ability to crowdsource question perturbations and their respective answers through independent LLM agents at scale. Our experiments culminated in the examination of 1.9 million question perturbations and 2.3 million answers. Furthermore, MultiQ&A shows that ensembled LLMs, such as ","authors_text":"Nicole Cho, William Watson","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-06T01:58:48Z","title":"MultiQ&A: An Analysis in Measuring Robustness via Automated Crowdsourcing of Question Perturbations and Answers"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.03711","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6d4c70f23f096ddd3151b3f138aeb15fda57184738c4adaa405070e38a365da3","target":"record","created_at":"2026-07-05T10:10:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"db1c8ef3ffb893c4585efd88007a9a82d43fd9cd09aa61715e0fa969289e34bb","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-02-06T01:58:48Z","title_canon_sha256":"edb6c7540631727f68bd37266a135b94364f0ee75ae1de87ba534fa8fb382b19"},"schema_version":"1.0","source":{"id":"2502.03711","kind":"arxiv","version":1}},"canonical_sha256":"0f0833bbcff253321432cb0cf47c3172e51c52d2578ea67ff5110dfb0b4fdfe0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0f0833bbcff253321432cb0cf47c3172e51c52d2578ea67ff5110dfb0b4fdfe0","first_computed_at":"2026-07-05T10:10:14.036589Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:10:14.036589Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"6mBSzKVctsktWYQc+xY1liMUgfvrgS0XlsK/+AiALi9jUbdrLq4Akwxxws3qdnbHPwtJWZbl5tntg5qduKpDAg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:10:14.037001Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.03711","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6d4c70f23f096ddd3151b3f138aeb15fda57184738c4adaa405070e38a365da3","sha256:e54dd0fffd6b26edaa4dfb76ae3597b8157be3d5cb3fdd8ed02643015b5b98d7"],"state_sha256":"ca084235b5d616b5bf60718c2d8ee2b37d32ec8101063c55a79308151fa7b838"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"wh6B2SfRS3ovmFzKOiC9CAzsMK0ToInj36OvOLjPqVwYkgr5r5jSpVSeO0ecXbbVfDaCthwjiwaN8uZpeQ/LBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-25T21:37:44.503587Z","bundle_sha256":"9945f5a6962ff01ed7089f0e04966b395a06279d56c8e59cb030f0a2f249e36c"}}