{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:E3E5YQVFQOX2S36HL2ZHEROPSO","short_pith_number":"pith:E3E5YQVF","canonical_record":{"source":{"id":"2301.12652","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-01-30T04:18:09Z","cross_cats_sorted":[],"title_canon_sha256":"fa983c86e0d5244a202381984860f5408f78ee6d60eb1d653ba566c65bd3d062","abstract_canon_sha256":"73a98e164e6404904005b5333a959035c0df5b588250f9770de74d53af6aac50"},"schema_version":"1.0"},"canonical_sha256":"26c9dc42a583afa96fc75eb27245cf938a2ff384cee2bb3a0697f22b966fdff9","source":{"kind":"arxiv","id":"2301.12652","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2301.12652","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"arxiv_version","alias_value":"2301.12652v4","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2301.12652","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"pith_short_12","alias_value":"E3E5YQVFQOX2","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"E3E5YQVFQOX2S36H","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"E3E5YQVF","created_at":"2026-05-18T12:33:33Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:E3E5YQVFQOX2S36HL2ZHEROPSO","target":"record","payload":{"canonical_record":{"source":{"id":"2301.12652","kind":"arxiv","version":4},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-01-30T04:18:09Z","cross_cats_sorted":[],"title_canon_sha256":"fa983c86e0d5244a202381984860f5408f78ee6d60eb1d653ba566c65bd3d062","abstract_canon_sha256":"73a98e164e6404904005b5333a959035c0df5b588250f9770de74d53af6aac50"},"schema_version":"1.0"},"canonical_sha256":"26c9dc42a583afa96fc75eb27245cf938a2ff384cee2bb3a0697f22b966fdff9","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:38:14.074836Z","signature_b64":"X1ljsf1f/n1uE84WWxHjdVpLx3ZRejtyxWNbAHMkESPOgzz+Fo21mh/c+ZAkkOPrrgSIu0HtvOnlvT91lAjdAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"26c9dc42a583afa96fc75eb27245cf938a2ff384cee2bb3a0697f22b966fdff9","last_reissued_at":"2026-05-17T23:38:14.073876Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:38:14.073876Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2301.12652","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GNHPys7sTfnj7rTWxn1HRTlPUk1LgzUCX+GWHekZfpH42Uc8uRMJIAuuIYHcQqMtnBidl5TzUohDnx2w2J+eDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-28T23:20:10.015852Z"},"content_sha256":"ec588dc4c5fb249acd10f41a27d673ae77d8a3169b1f581241bd6f28f85a1b6d","schema_version":"1.0","event_id":"sha256:ec588dc4c5fb249acd10f41a27d673ae77d8a3169b1f581241bd6f28f85a1b6d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:E3E5YQVFQOX2S36HL2ZHEROPSO","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"REPLUG: Retrieval-Augmented Black-Box Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"REPLUG augments frozen black-box LMs like GPT-3 with a tunable retriever by prepending documents and training the retriever on the LM's own predictions.","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Luke Zettlemoyer, Michihiro Yasunaga, Mike Lewis, Minjoon Seo, Rich James, Sewon Min, Weijia Shi, Wen-tau Yih","submitted_at":"2023-01-30T04:18:09Z","abstract_excerpt":"We introduce REPLUG, a retrieval-augmented language modeling framework that treats the language model (LM) as a black box and augments it with a tuneable retrieval model. Unlike prior retrieval-augmented LMs that train language models with special cross attention mechanisms to encode the retrieved text, REPLUG simply prepends retrieved documents to the input for the frozen black-box LM. This simple design can be easily applied to any existing retrieval and language models. Furthermore, we show that the LM can be used to supervise the retrieval model, which can then find documents that help the"},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"REPLUG with the tuned retriever significantly improves the performance of GPT-3 (175B) on language modeling by 6.3%, as well as the performance of Codex on five-shot MMLU by 5.1%.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"That the frozen LM can reliably supervise the retriever to surface documents that genuinely improve its own predictions without introducing evaluation bias or requiring task-specific labels.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"REPLUG improves frozen black-box LMs by prepending LM-supervised retrieved documents, delivering 6.3% better language modeling on GPT-3 and 5.1% better five-shot MMLU on Codex.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"REPLUG augments frozen black-box LMs like GPT-3 with a tunable retriever by prepending documents and training the retriever on the LM's own predictions.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"2a2829e2da3e9ed4e259953993ffcf3ef8a5bd1285171e1f950ff98a859da15d"},"source":{"id":"2301.12652","kind":"arxiv","version":4},"verdict":{"id":"faf056f5-ee32-4fd5-acf5-500f1a853d94","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-17T12:36:41.264902Z","strongest_claim":"REPLUG with the tuned retriever significantly improves the performance of GPT-3 (175B) on language modeling by 6.3%, as well as the performance of Codex on five-shot MMLU by 5.1%.","one_line_summary":"REPLUG improves frozen black-box LMs by prepending LM-supervised retrieved documents, delivering 6.3% better language modeling on GPT-3 and 5.1% better five-shot MMLU on Codex.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"That the frozen LM can reliably supervise the retriever to surface documents that genuinely improve its own predictions without introducing evaluation bias or requiring task-specific labels.","pith_extraction_headline":"REPLUG augments frozen black-box LMs like GPT-3 with a tunable retriever by prepending documents and training the retriever on the LM's own predictions."},"references":{"count":300,"sample":[{"doi":"","year":2022,"title":"International Conference on Machine Learning , pages=","work_id":"9d7db47e-b04c-42b8-820e-fd314f9bb81f","ref_index":1,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":2017,"title":"5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24-26, 2017, Conference Track Proceedings , year=","work_id":"fbe7703f-835e-477c-93c1-28754e816d7a","ref_index":2,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"Meta AI , year=","work_id":"137f666a-607a-4f78-9f92-ca9fd1164701","ref_index":3,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"Yuan 1.0: Large- scale pre-trained language model in zero-shot and few-shot learning","work_id":"2492c13b-980b-4c7a-ab9a-9c286e174189","ref_index":4,"cited_arxiv_id":"","is_internal_anchor":false},{"doi":"","year":null,"title":"Language Models are Few-Shot Learners , url =","work_id":"518a19dc-59e5-4fa9-b31b-b2c1f0d6676c","ref_index":5,"cited_arxiv_id":"","is_internal_anchor":false}],"resolved_work":300,"snapshot_sha256":"413cc37821798de9e7ff04790bde98c21192dc0e14bdfd7f38640a0dd109c1ed","internal_anchors":16},"formal_canon":{"evidence_count":1,"snapshot_sha256":"a9ed69d09e8681af6e55c6a57a23ad9dad30b07155fe3a8ac418508c5dd79ddd"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"faf056f5-ee32-4fd5-acf5-500f1a853d94"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:38:14Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3AyNpd7kJpSb0AJJ9jFHkCWo8bTzyrRC4gU4UY++RFTOLY/AakKqVke97dKOfYwIBfsiXjd15ilIvpc5uNu+Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-28T23:20:10.016616Z"},"content_sha256":"75eea78433266eea60d224f35dae597a77d187f357b317acb2b8f6816e720d44","schema_version":"1.0","event_id":"sha256:75eea78433266eea60d224f35dae597a77d187f357b317acb2b8f6816e720d44"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/E3E5YQVFQOX2S36HL2ZHEROPSO/bundle.json","state_url":"https://pith.science/pith/E3E5YQVFQOX2S36HL2ZHEROPSO/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/E3E5YQVFQOX2S36HL2ZHEROPSO/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-28T23:20:10Z","links":{"resolver":"https://pith.science/pith/E3E5YQVFQOX2S36HL2ZHEROPSO","bundle":"https://pith.science/pith/E3E5YQVFQOX2S36HL2ZHEROPSO/bundle.json","state":"https://pith.science/pith/E3E5YQVFQOX2S36HL2ZHEROPSO/state.json","well_known_bundle":"https://pith.science/.well-known/pith/E3E5YQVFQOX2S36HL2ZHEROPSO/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:E3E5YQVFQOX2S36HL2ZHEROPSO","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"73a98e164e6404904005b5333a959035c0df5b588250f9770de74d53af6aac50","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-01-30T04:18:09Z","title_canon_sha256":"fa983c86e0d5244a202381984860f5408f78ee6d60eb1d653ba566c65bd3d062"},"schema_version":"1.0","source":{"id":"2301.12652","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2301.12652","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"arxiv_version","alias_value":"2301.12652v4","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2301.12652","created_at":"2026-05-17T23:38:14Z"},{"alias_kind":"pith_short_12","alias_value":"E3E5YQVFQOX2","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_16","alias_value":"E3E5YQVFQOX2S36H","created_at":"2026-05-18T12:33:33Z"},{"alias_kind":"pith_short_8","alias_value":"E3E5YQVF","created_at":"2026-05-18T12:33:33Z"}],"graph_snapshots":[{"event_id":"sha256:75eea78433266eea60d224f35dae597a77d187f357b317acb2b8f6816e720d44","target":"graph","created_at":"2026-05-17T23:38:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"REPLUG with the tuned retriever significantly improves the performance of GPT-3 (175B) on language modeling by 6.3%, as well as the performance of Codex on five-shot MMLU by 5.1%."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"That the frozen LM can reliably supervise the retriever to surface documents that genuinely improve its own predictions without introducing evaluation bias or requiring task-specific labels."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"REPLUG improves frozen black-box LMs by prepending LM-supervised retrieved documents, delivering 6.3% better language modeling on GPT-3 and 5.1% better five-shot MMLU on Codex."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"REPLUG augments frozen black-box LMs like GPT-3 with a tunable retriever by prepending documents and training the retriever on the LM's own predictions."}],"snapshot_sha256":"2a2829e2da3e9ed4e259953993ffcf3ef8a5bd1285171e1f950ff98a859da15d"},"formal_canon":{"evidence_count":1,"snapshot_sha256":"a9ed69d09e8681af6e55c6a57a23ad9dad30b07155fe3a8ac418508c5dd79ddd"},"paper":{"abstract_excerpt":"We introduce REPLUG, a retrieval-augmented language modeling framework that treats the language model (LM) as a black box and augments it with a tuneable retrieval model. Unlike prior retrieval-augmented LMs that train language models with special cross attention mechanisms to encode the retrieved text, REPLUG simply prepends retrieved documents to the input for the frozen black-box LM. This simple design can be easily applied to any existing retrieval and language models. Furthermore, we show that the LM can be used to supervise the retrieval model, which can then find documents that help the","authors_text":"Luke Zettlemoyer, Michihiro Yasunaga, Mike Lewis, Minjoon Seo, Rich James, Sewon Min, Weijia Shi, Wen-tau Yih","cross_cats":[],"headline":"REPLUG augments frozen black-box LMs like GPT-3 with a tunable retriever by prepending documents and training the retriever on the LM's own predictions.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-01-30T04:18:09Z","title":"REPLUG: Retrieval-Augmented Black-Box Language Models"},"references":{"count":300,"internal_anchors":16,"resolved_work":300,"sample":[{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":1,"title":"International Conference on Machine Learning , pages=","work_id":"9d7db47e-b04c-42b8-820e-fd314f9bb81f","year":2022},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":2,"title":"5th International Conference on Learning Representations, ICLR 2017, Toulon, France, April 24-26, 2017, Conference Track Proceedings , year=","work_id":"fbe7703f-835e-477c-93c1-28754e816d7a","year":2017},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":3,"title":"Meta AI , year=","work_id":"137f666a-607a-4f78-9f92-ca9fd1164701","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":4,"title":"Yuan 1.0: Large- scale pre-trained language model in zero-shot and few-shot learning","work_id":"2492c13b-980b-4c7a-ab9a-9c286e174189","year":null},{"cited_arxiv_id":"","doi":"","is_internal_anchor":false,"ref_index":5,"title":"Language Models are Few-Shot Learners , url =","work_id":"518a19dc-59e5-4fa9-b31b-b2c1f0d6676c","year":null}],"snapshot_sha256":"413cc37821798de9e7ff04790bde98c21192dc0e14bdfd7f38640a0dd109c1ed"},"source":{"id":"2301.12652","kind":"arxiv","version":4},"verdict":{"created_at":"2026-05-17T12:36:41.264902Z","id":"faf056f5-ee32-4fd5-acf5-500f1a853d94","model_set":{"reader":"grok-4.3"},"one_line_summary":"REPLUG improves frozen black-box LMs by prepending LM-supervised retrieved documents, delivering 6.3% better language modeling on GPT-3 and 5.1% better five-shot MMLU on Codex.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"REPLUG augments frozen black-box LMs like GPT-3 with a tunable retriever by prepending documents and training the retriever on the LM's own predictions.","strongest_claim":"REPLUG with the tuned retriever significantly improves the performance of GPT-3 (175B) on language modeling by 6.3%, as well as the performance of Codex on five-shot MMLU by 5.1%.","weakest_assumption":"That the frozen LM can reliably supervise the retriever to surface documents that genuinely improve its own predictions without introducing evaluation bias or requiring task-specific labels."}},"verdict_id":"faf056f5-ee32-4fd5-acf5-500f1a853d94"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ec588dc4c5fb249acd10f41a27d673ae77d8a3169b1f581241bd6f28f85a1b6d","target":"record","created_at":"2026-05-17T23:38:14Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"73a98e164e6404904005b5333a959035c0df5b588250f9770de74d53af6aac50","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-01-30T04:18:09Z","title_canon_sha256":"fa983c86e0d5244a202381984860f5408f78ee6d60eb1d653ba566c65bd3d062"},"schema_version":"1.0","source":{"id":"2301.12652","kind":"arxiv","version":4}},"canonical_sha256":"26c9dc42a583afa96fc75eb27245cf938a2ff384cee2bb3a0697f22b966fdff9","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"26c9dc42a583afa96fc75eb27245cf938a2ff384cee2bb3a0697f22b966fdff9","first_computed_at":"2026-05-17T23:38:14.073876Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:38:14.073876Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"X1ljsf1f/n1uE84WWxHjdVpLx3ZRejtyxWNbAHMkESPOgzz+Fo21mh/c+ZAkkOPrrgSIu0HtvOnlvT91lAjdAw==","signature_status":"signed_v1","signed_at":"2026-05-17T23:38:14.074836Z","signed_message":"canonical_sha256_bytes"},"source_id":"2301.12652","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ec588dc4c5fb249acd10f41a27d673ae77d8a3169b1f581241bd6f28f85a1b6d","sha256:75eea78433266eea60d224f35dae597a77d187f357b317acb2b8f6816e720d44"],"state_sha256":"6d53a8c4f485d599a6322ee2f55649c6a2fc9246d62830acd2c96d766eaa951e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"O2aAH1sOY243WKTl/iKa8fBQQeKnB+x0zJt4a4nl20+J7OqnTTqfgsCJ35kHiMDh5viFx8GKY0yQHWKfXY/wBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-28T23:20:10.021436Z","bundle_sha256":"fa0d1f9bd6d282be3913027d3af5132c582eba564ce95f27e31aadfefcb11136"}}