{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2022:YBPYMZDX5LQHG3LKCHKX72HPWN","short_pith_number":"pith:YBPYMZDX","schema_version":"1.0","canonical_sha256":"c05f866477eae0736d6a11d57fe8efb3568cc29e022824335655c3f43eb4f84e","source":{"kind":"arxiv","id":"2209.14389","version":2},"attestation_state":"computed","paper":{"title":"Downstream Datasets Make Surprisingly Good Pretraining Corpora","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Jeffrey P. Bigham, Kundan Krishna, Saurabh Garg, Zachary C. Lipton","submitted_at":"2022-09-28T19:28:43Z","abstract_excerpt":"For most natural language processing tasks, the dominant practice is to finetune large pretrained transformer models (e.g., BERT) using smaller downstream datasets. Despite the success of this approach, it remains unclear to what extent these gains are attributable to the massive background corpora employed for pretraining versus to the pretraining objectives themselves. This paper introduces a large-scale study of self-pretraining, where the same (downstream) training data is used for both pretraining and finetuning. In experiments addressing both ELECTRA and RoBERTa models and 10 distinct do"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2209.14389","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2022-09-28T19:28:43Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"5cec3daee1b0319c866d5f36ec225a9c01e63c5f90b1860825f5e73d51ac4280","abstract_canon_sha256":"a5de17431b08bc64fb0e487511b53700fb817cb55b1d396a1a49b42d78a75dcc"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:14:05.320962Z","signature_b64":"kWK56nEeIFUS+rQ8xh25zXcizdF4LY76wwnHHHdQ5YvYiKZQwKBOaWx5Wi3aNz0q38cZnCtzbFuuFySfJ9GrAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c05f866477eae0736d6a11d57fe8efb3568cc29e022824335655c3f43eb4f84e","last_reissued_at":"2026-07-05T06:14:05.320473Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:14:05.320473Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Downstream Datasets Make Surprisingly Good Pretraining Corpora","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.CL","authors_text":"Jeffrey P. Bigham, Kundan Krishna, Saurabh Garg, Zachary C. Lipton","submitted_at":"2022-09-28T19:28:43Z","abstract_excerpt":"For most natural language processing tasks, the dominant practice is to finetune large pretrained transformer models (e.g., BERT) using smaller downstream datasets. Despite the success of this approach, it remains unclear to what extent these gains are attributable to the massive background corpora employed for pretraining versus to the pretraining objectives themselves. This paper introduces a large-scale study of self-pretraining, where the same (downstream) training data is used for both pretraining and finetuning. In experiments addressing both ELECTRA and RoBERTa models and 10 distinct do"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2209.14389","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2209.14389/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2209.14389","created_at":"2026-07-05T06:14:05.320535+00:00"},{"alias_kind":"arxiv_version","alias_value":"2209.14389v2","created_at":"2026-07-05T06:14:05.320535+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2209.14389","created_at":"2026-07-05T06:14:05.320535+00:00"},{"alias_kind":"pith_short_12","alias_value":"YBPYMZDX5LQH","created_at":"2026-07-05T06:14:05.320535+00:00"},{"alias_kind":"pith_short_16","alias_value":"YBPYMZDX5LQHG3LK","created_at":"2026-07-05T06:14:05.320535+00:00"},{"alias_kind":"pith_short_8","alias_value":"YBPYMZDX","created_at":"2026-07-05T06:14:05.320535+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2211.09085","citing_title":"Galactica: A Large Language Model for Science","ref_index":79,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN","json":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN.json","graph_json":"https://pith.science/api/pith-number/YBPYMZDX5LQHG3LKCHKX72HPWN/graph.json","events_json":"https://pith.science/api/pith-number/YBPYMZDX5LQHG3LKCHKX72HPWN/events.json","paper":"https://pith.science/paper/YBPYMZDX"},"agent_actions":{"view_html":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN","download_json":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN.json","view_paper":"https://pith.science/paper/YBPYMZDX","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2209.14389&json=true","fetch_graph":"https://pith.science/api/pith-number/YBPYMZDX5LQHG3LKCHKX72HPWN/graph.json","fetch_events":"https://pith.science/api/pith-number/YBPYMZDX5LQHG3LKCHKX72HPWN/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN/action/timestamp_anchor","attest_storage":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN/action/storage_attestation","attest_author":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN/action/author_attestation","sign_citation":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN/action/citation_signature","submit_replication":"https://pith.science/pith/YBPYMZDX5LQHG3LKCHKX72HPWN/action/replication_record"}},"created_at":"2026-07-05T06:14:05.320535+00:00","updated_at":"2026-07-05T06:14:05.320535+00:00"}