{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:GSX7HHWEDWXRHMRET46CUMNWPQ","short_pith_number":"pith:GSX7HHWE","canonical_record":{"source":{"id":"2408.08656","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-16T10:45:45Z","cross_cats_sorted":[],"title_canon_sha256":"f84d27a21cd3a961010e102077db770c841ba6de37c10d1cca855bd7e117d132","abstract_canon_sha256":"640ec1f38dba40ce91752447b3f86f6d09a04b20a34663ce3c4037eeae2cb7e7"},"schema_version":"1.0"},"canonical_sha256":"34aff39ec41daf13b2249f3c2a31b67c15ee7d9446192a75640c99e50dc786cf","source":{"kind":"arxiv","id":"2408.08656","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.08656","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"arxiv_version","alias_value":"2408.08656v2","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.08656","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"pith_short_12","alias_value":"GSX7HHWEDWXR","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"pith_short_16","alias_value":"GSX7HHWEDWXRHMRE","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"pith_short_8","alias_value":"GSX7HHWE","created_at":"2026-07-05T10:18:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:GSX7HHWEDWXRHMRET46CUMNWPQ","target":"record","payload":{"canonical_record":{"source":{"id":"2408.08656","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-16T10:45:45Z","cross_cats_sorted":[],"title_canon_sha256":"f84d27a21cd3a961010e102077db770c841ba6de37c10d1cca855bd7e117d132","abstract_canon_sha256":"640ec1f38dba40ce91752447b3f86f6d09a04b20a34663ce3c4037eeae2cb7e7"},"schema_version":"1.0"},"canonical_sha256":"34aff39ec41daf13b2249f3c2a31b67c15ee7d9446192a75640c99e50dc786cf","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:18:40.525731Z","signature_b64":"i1TkOGh4gYjWlgltvoXVLcQXZHzo9ClWVOd+zhdkOgBg1beL6E66pjA5pLqlTm8nvc7YMowCaDtlbrlUm/kJAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"34aff39ec41daf13b2249f3c2a31b67c15ee7d9446192a75640c99e50dc786cf","last_reissued_at":"2026-07-05T10:18:40.525222Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:18:40.525222Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.08656","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:18:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nlVviJiXd8Y18F9Cp4/aZhFIUD4R6tGTFhxTcwTTA3NhnblXRNVU5lgddmRqUoV70tVtZihAS/qirqWsJV4vAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T02:58:31.813058Z"},"content_sha256":"599637e1699a34242b147f03dc221d5b5cbf83a6d6bb626ed2948e6b7c56cedc","schema_version":"1.0","event_id":"sha256:599637e1699a34242b147f03dc221d5b5cbf83a6d6bb626ed2948e6b7c56cedc"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:GSX7HHWEDWXRHMRET46CUMNWPQ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"LLMs Are Biased Towards Output Formats! Systematically Evaluating and Mitigating Output Format Bias of LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Do Xuan Long, Hai Nguyen Ngoc, Hieu Dao, Kenji Kawaguchi, Min-Yen Kan, Nancy F. Chen, Shafiq Joty, Tiviatis Sim","submitted_at":"2024-08-16T10:45:45Z","abstract_excerpt":"We present the first systematic evaluation examining format bias in performance of large language models (LLMs). Our approach distinguishes between two categories of an evaluation metric under format constraints to reliably and accurately assess performance: one measures performance when format constraints are adhered to, while the other evaluates performance regardless of constraint adherence. We then define a metric for measuring the format bias of LLMs and establish effective strategies to reduce it. Subsequently, we present our empirical format bias evaluation spanning four commonly used c"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.08656","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.08656/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:18:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"N6fcEiPSzUscS8HSSGTmHKIA+JJ+Rke70E3JACmv3arDBroYNmA3CCbHi0fCC7G6SN13lw7Aj6pTKahZ4mlNAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T02:58:31.813563Z"},"content_sha256":"569401361214cc5f3b9e985519f083a4348dcd26d9ff4338d26eb91f3f7b9f28","schema_version":"1.0","event_id":"sha256:569401361214cc5f3b9e985519f083a4348dcd26d9ff4338d26eb91f3f7b9f28"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GSX7HHWEDWXRHMRET46CUMNWPQ/bundle.json","state_url":"https://pith.science/pith/GSX7HHWEDWXRHMRET46CUMNWPQ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GSX7HHWEDWXRHMRET46CUMNWPQ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T02:58:31Z","links":{"resolver":"https://pith.science/pith/GSX7HHWEDWXRHMRET46CUMNWPQ","bundle":"https://pith.science/pith/GSX7HHWEDWXRHMRET46CUMNWPQ/bundle.json","state":"https://pith.science/pith/GSX7HHWEDWXRHMRET46CUMNWPQ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GSX7HHWEDWXRHMRET46CUMNWPQ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GSX7HHWEDWXRHMRET46CUMNWPQ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"640ec1f38dba40ce91752447b3f86f6d09a04b20a34663ce3c4037eeae2cb7e7","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-16T10:45:45Z","title_canon_sha256":"f84d27a21cd3a961010e102077db770c841ba6de37c10d1cca855bd7e117d132"},"schema_version":"1.0","source":{"id":"2408.08656","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.08656","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"arxiv_version","alias_value":"2408.08656v2","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.08656","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"pith_short_12","alias_value":"GSX7HHWEDWXR","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"pith_short_16","alias_value":"GSX7HHWEDWXRHMRE","created_at":"2026-07-05T10:18:40Z"},{"alias_kind":"pith_short_8","alias_value":"GSX7HHWE","created_at":"2026-07-05T10:18:40Z"}],"graph_snapshots":[{"event_id":"sha256:569401361214cc5f3b9e985519f083a4348dcd26d9ff4338d26eb91f3f7b9f28","target":"graph","created_at":"2026-07-05T10:18:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.08656/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We present the first systematic evaluation examining format bias in performance of large language models (LLMs). Our approach distinguishes between two categories of an evaluation metric under format constraints to reliably and accurately assess performance: one measures performance when format constraints are adhered to, while the other evaluates performance regardless of constraint adherence. We then define a metric for measuring the format bias of LLMs and establish effective strategies to reduce it. Subsequently, we present our empirical format bias evaluation spanning four commonly used c","authors_text":"Do Xuan Long, Hai Nguyen Ngoc, Hieu Dao, Kenji Kawaguchi, Min-Yen Kan, Nancy F. Chen, Shafiq Joty, Tiviatis Sim","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-16T10:45:45Z","title":"LLMs Are Biased Towards Output Formats! Systematically Evaluating and Mitigating Output Format Bias of LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.08656","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:599637e1699a34242b147f03dc221d5b5cbf83a6d6bb626ed2948e6b7c56cedc","target":"record","created_at":"2026-07-05T10:18:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"640ec1f38dba40ce91752447b3f86f6d09a04b20a34663ce3c4037eeae2cb7e7","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-16T10:45:45Z","title_canon_sha256":"f84d27a21cd3a961010e102077db770c841ba6de37c10d1cca855bd7e117d132"},"schema_version":"1.0","source":{"id":"2408.08656","kind":"arxiv","version":2}},"canonical_sha256":"34aff39ec41daf13b2249f3c2a31b67c15ee7d9446192a75640c99e50dc786cf","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"34aff39ec41daf13b2249f3c2a31b67c15ee7d9446192a75640c99e50dc786cf","first_computed_at":"2026-07-05T10:18:40.525222Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:18:40.525222Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"i1TkOGh4gYjWlgltvoXVLcQXZHzo9ClWVOd+zhdkOgBg1beL6E66pjA5pLqlTm8nvc7YMowCaDtlbrlUm/kJAA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:18:40.525731Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.08656","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:599637e1699a34242b147f03dc221d5b5cbf83a6d6bb626ed2948e6b7c56cedc","sha256:569401361214cc5f3b9e985519f083a4348dcd26d9ff4338d26eb91f3f7b9f28"],"state_sha256":"f7aa81ef89ee0c28dbef214cb66f4d67736ec812ec1548efdc1a1a3ebc91f3b9"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XyME5bw29AiVvjtZY9DnsDkB9e+b+XWTPPZlbCJLmfaXZq1fkCmsMqvUurvHnDMLKVJh80AyNIR6BUZe70SZAw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T02:58:31.818998Z","bundle_sha256":"011ca054490dd327ebaf59fc49e56d7ba90134ffaf9ef2205b8e9208263420db"}}