{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:6ZWAAYZGQRFOKFLRGL5TIMVMWO","short_pith_number":"pith:6ZWAAYZG","schema_version":"1.0","canonical_sha256":"f66c006326844ae5157132fb3432acb3a31cb81d8f127376956e753177ec864d","source":{"kind":"arxiv","id":"2508.00109","version":1},"attestation_state":"computed","paper":{"title":"FACTORY: A Challenging Human-Verified Prompt Set for Long-Form Factuality","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Adina Williams, Gargi Ghosh, Mingda Chen, Scott Yih, Xilun Chen, Yang Li","submitted_at":"2025-07-31T19:00:11Z","abstract_excerpt":"Long-form factuality evaluation assesses the ability of models to generate accurate, comprehensive responses to short prompts. Existing benchmarks often lack human verification, leading to potential quality issues. To address this limitation, we introduce FACTORY, a large-scale, human-verified prompt set. Developed using a model-in-the-loop approach and refined by humans, FACTORY includes challenging prompts that are fact-seeking, answerable, and unambiguous. We conduct human evaluations on 6 state-of-the-art language models using FACTORY and existing datasets. Our results show that FACTORY is"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2508.00109","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-31T19:00:11Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"e2043ef2ec7aacc8ea8edd3eed97dba617a24c2fafba1ad04e83f92a2799e1d0","abstract_canon_sha256":"888de1387a90ea8979d5c8c39c89441b967de9a8736e6e07ac0b9660f109105e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:46:49.041471Z","signature_b64":"xgjzhGyc2lBiaTAiylsROzY4OaW5Pqh1d5BlWKZgb5cIcIK9CPRCPU2nlNtKTRaO39m5f+IrmQjKGQggbdaZBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f66c006326844ae5157132fb3432acb3a31cb81d8f127376956e753177ec864d","last_reissued_at":"2026-07-05T11:46:49.041004Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:46:49.041004Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"FACTORY: A Challenging Human-Verified Prompt Set for Long-Form Factuality","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Adina Williams, Gargi Ghosh, Mingda Chen, Scott Yih, Xilun Chen, Yang Li","submitted_at":"2025-07-31T19:00:11Z","abstract_excerpt":"Long-form factuality evaluation assesses the ability of models to generate accurate, comprehensive responses to short prompts. Existing benchmarks often lack human verification, leading to potential quality issues. To address this limitation, we introduce FACTORY, a large-scale, human-verified prompt set. Developed using a model-in-the-loop approach and refined by humans, FACTORY includes challenging prompts that are fact-seeking, answerable, and unambiguous. We conduct human evaluations on 6 state-of-the-art language models using FACTORY and existing datasets. Our results show that FACTORY is"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2508.00109","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2508.00109/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2508.00109","created_at":"2026-07-05T11:46:49.041051+00:00"},{"alias_kind":"arxiv_version","alias_value":"2508.00109v1","created_at":"2026-07-05T11:46:49.041051+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2508.00109","created_at":"2026-07-05T11:46:49.041051+00:00"},{"alias_kind":"pith_short_12","alias_value":"6ZWAAYZGQRFO","created_at":"2026-07-05T11:46:49.041051+00:00"},{"alias_kind":"pith_short_16","alias_value":"6ZWAAYZGQRFOKFLR","created_at":"2026-07-05T11:46:49.041051+00:00"},{"alias_kind":"pith_short_8","alias_value":"6ZWAAYZG","created_at":"2026-07-05T11:46:49.041051+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2605.01749","citing_title":"Only Say What You Know: Calibration-Aware Generation for Long-Form Factuality","ref_index":21,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO","json":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO.json","graph_json":"https://pith.science/api/pith-number/6ZWAAYZGQRFOKFLRGL5TIMVMWO/graph.json","events_json":"https://pith.science/api/pith-number/6ZWAAYZGQRFOKFLRGL5TIMVMWO/events.json","paper":"https://pith.science/paper/6ZWAAYZG"},"agent_actions":{"view_html":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO","download_json":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO.json","view_paper":"https://pith.science/paper/6ZWAAYZG","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2508.00109&json=true","fetch_graph":"https://pith.science/api/pith-number/6ZWAAYZGQRFOKFLRGL5TIMVMWO/graph.json","fetch_events":"https://pith.science/api/pith-number/6ZWAAYZGQRFOKFLRGL5TIMVMWO/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO/action/timestamp_anchor","attest_storage":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO/action/storage_attestation","attest_author":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO/action/author_attestation","sign_citation":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO/action/citation_signature","submit_replication":"https://pith.science/pith/6ZWAAYZGQRFOKFLRGL5TIMVMWO/action/replication_record"}},"created_at":"2026-07-05T11:46:49.041051+00:00","updated_at":"2026-07-05T11:46:49.041051+00:00"}