{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:NWXJW3XJAAPX6VFVDKMTUVFFR3","short_pith_number":"pith:NWXJW3XJ","schema_version":"1.0","canonical_sha256":"6dae9b6ee9001f7f54b51a993a54a58ed6aa7f238f62eb02cebd04779f666558","source":{"kind":"arxiv","id":"2310.07712","version":2},"attestation_state":"computed","paper":{"title":"Found in the Middle: Permutation Self-Consistency Improves Listwise Ranking in Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Ferhan Ture, Jimmy Lin, Raphael Tang, Xinyu Zhang, Xueguang Ma","submitted_at":"2023-10-11T17:59:02Z","abstract_excerpt":"Large language models (LLMs) exhibit positional bias in how they use context, which especially complicates listwise ranking. To address this, we propose permutation self-consistency, a form of self-consistency over ranking list outputs of black-box LLMs. Our key idea is to marginalize out different list orders in the prompt to produce an order-independent ranking with less positional bias. First, given some input prompt, we repeatedly shuffle the list in the prompt and pass it through the LLM while holding the instructions the same. Next, we aggregate the resulting sample of rankings by comput"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2310.07712","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-10-11T17:59:02Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"f0f4ee98be2d03985b1f0f742118714be3898733c51b8f75b5625bc4913c7aef","abstract_canon_sha256":"6ee9882ac4ef4aac75c0e68905f5809efce2a414f01caddca66a92c2e9f51b0c"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:10:18.764074Z","signature_b64":"/pUvguMSd5xOJ3qI+jwDY7QuSUyg+sVdqN1MPMklGw4PpR8L3NMX3RnjXhmnXfHfuWutQDdgv6nEDRcEvnt+BA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6dae9b6ee9001f7f54b51a993a54a58ed6aa7f238f62eb02cebd04779f666558","last_reissued_at":"2026-07-05T08:10:18.763633Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:10:18.763633Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Found in the Middle: Permutation Self-Consistency Improves Listwise Ranking in Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Ferhan Ture, Jimmy Lin, Raphael Tang, Xinyu Zhang, Xueguang Ma","submitted_at":"2023-10-11T17:59:02Z","abstract_excerpt":"Large language models (LLMs) exhibit positional bias in how they use context, which especially complicates listwise ranking. To address this, we propose permutation self-consistency, a form of self-consistency over ranking list outputs of black-box LLMs. Our key idea is to marginalize out different list orders in the prompt to produce an order-independent ranking with less positional bias. First, given some input prompt, we repeatedly shuffle the list in the prompt and pass it through the LLM while holding the instructions the same. Next, we aggregate the resulting sample of rankings by comput"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2310.07712","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2310.07712/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2310.07712","created_at":"2026-07-05T08:10:18.763696+00:00"},{"alias_kind":"arxiv_version","alias_value":"2310.07712v2","created_at":"2026-07-05T08:10:18.763696+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2310.07712","created_at":"2026-07-05T08:10:18.763696+00:00"},{"alias_kind":"pith_short_12","alias_value":"NWXJW3XJAAPX","created_at":"2026-07-05T08:10:18.763696+00:00"},{"alias_kind":"pith_short_16","alias_value":"NWXJW3XJAAPX6VFV","created_at":"2026-07-05T08:10:18.763696+00:00"},{"alias_kind":"pith_short_8","alias_value":"NWXJW3XJ","created_at":"2026-07-05T08:10:18.763696+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":3,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.17237","citing_title":"HeadRank: Decoding-Free Passage Reranking via Preference-Aligned Attention Heads","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2312.02724","citing_title":"RankZephyr: Effective and Robust Zero-Shot Listwise Reranking is a Breeze!","ref_index":36,"is_internal_anchor":false},{"citing_arxiv_id":"2604.17237","citing_title":"HeadRank: Decoding-Free Passage Reranking via Preference-Aligned Attention Heads","ref_index":37,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3","json":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3.json","graph_json":"https://pith.science/api/pith-number/NWXJW3XJAAPX6VFVDKMTUVFFR3/graph.json","events_json":"https://pith.science/api/pith-number/NWXJW3XJAAPX6VFVDKMTUVFFR3/events.json","paper":"https://pith.science/paper/NWXJW3XJ"},"agent_actions":{"view_html":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3","download_json":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3.json","view_paper":"https://pith.science/paper/NWXJW3XJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2310.07712&json=true","fetch_graph":"https://pith.science/api/pith-number/NWXJW3XJAAPX6VFVDKMTUVFFR3/graph.json","fetch_events":"https://pith.science/api/pith-number/NWXJW3XJAAPX6VFVDKMTUVFFR3/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3/action/timestamp_anchor","attest_storage":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3/action/storage_attestation","attest_author":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3/action/author_attestation","sign_citation":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3/action/citation_signature","submit_replication":"https://pith.science/pith/NWXJW3XJAAPX6VFVDKMTUVFFR3/action/replication_record"}},"created_at":"2026-07-05T08:10:18.763696+00:00","updated_at":"2026-07-05T08:10:18.763696+00:00"}