{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2021:2QZUVCPJ6KWPHI4F32X76QOFNS","short_pith_number":"pith:2QZUVCPJ","schema_version":"1.0","canonical_sha256":"d4334a89e9f2acf3a385deafff41c56cb7251dd545d0d339a27825417aeadcdb","source":{"kind":"arxiv","id":"2107.12460","version":1},"attestation_state":"computed","paper":{"title":"Don't Sweep your Learning Rate under the Rug: A Closer Look at Cross-modal Transfer of Pretrained Transformers","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Danielle Rothermel, Jakob Foerster, Margaret Li, Tim Rockt\\\"aschel","submitted_at":"2021-07-26T20:20:48Z","abstract_excerpt":"Self-supervised pre-training of large-scale transformer models on text corpora followed by finetuning has achieved state-of-the-art on a number of natural language processing tasks. Recently, Lu et al. (2021, arXiv:2103.05247) claimed that frozen pretrained transformers (FPTs) match or outperform training from scratch as well as unfrozen (fine-tuned) pretrained transformers in a set of transfer tasks to other modalities. In our work, we find that this result is, in fact, an artifact of not tuning the learning rates. After carefully redesigning the empirical setup, we find that when tuning lear"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2107.12460","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2021-07-26T20:20:48Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5689a0f08cc850b9896c6576dc7698d2e6cdefe157eaaee47b9c5b02d7b53e0f","abstract_canon_sha256":"109701b7200f7d50adf510d1cc5f609d164342f6367522c4630e9b308025a8bf"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:00:56.511092Z","signature_b64":"fnPfRM50uX2fBoeR3efqw+z6CrnxxERWOy50wFCgDcnpKn1xCBFHkmQ1shsrJrmCGPvpwSkfsUrnv5rVAyJ4Ag==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"d4334a89e9f2acf3a385deafff41c56cb7251dd545d0d339a27825417aeadcdb","last_reissued_at":"2026-07-05T03:00:56.510676Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:00:56.510676Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Don't Sweep your Learning Rate under the Rug: A Closer Look at Cross-modal Transfer of Pretrained Transformers","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.LG","authors_text":"Danielle Rothermel, Jakob Foerster, Margaret Li, Tim Rockt\\\"aschel","submitted_at":"2021-07-26T20:20:48Z","abstract_excerpt":"Self-supervised pre-training of large-scale transformer models on text corpora followed by finetuning has achieved state-of-the-art on a number of natural language processing tasks. Recently, Lu et al. (2021, arXiv:2103.05247) claimed that frozen pretrained transformers (FPTs) match or outperform training from scratch as well as unfrozen (fine-tuned) pretrained transformers in a set of transfer tasks to other modalities. In our work, we find that this result is, in fact, an artifact of not tuning the learning rates. After carefully redesigning the empirical setup, we find that when tuning lear"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2107.12460","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2107.12460/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2107.12460","created_at":"2026-07-05T03:00:56.510741+00:00"},{"alias_kind":"arxiv_version","alias_value":"2107.12460v1","created_at":"2026-07-05T03:00:56.510741+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2107.12460","created_at":"2026-07-05T03:00:56.510741+00:00"},{"alias_kind":"pith_short_12","alias_value":"2QZUVCPJ6KWP","created_at":"2026-07-05T03:00:56.510741+00:00"},{"alias_kind":"pith_short_16","alias_value":"2QZUVCPJ6KWPHI4F","created_at":"2026-07-05T03:00:56.510741+00:00"},{"alias_kind":"pith_short_8","alias_value":"2QZUVCPJ","created_at":"2026-07-05T03:00:56.510741+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS","json":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS.json","graph_json":"https://pith.science/api/pith-number/2QZUVCPJ6KWPHI4F32X76QOFNS/graph.json","events_json":"https://pith.science/api/pith-number/2QZUVCPJ6KWPHI4F32X76QOFNS/events.json","paper":"https://pith.science/paper/2QZUVCPJ"},"agent_actions":{"view_html":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS","download_json":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS.json","view_paper":"https://pith.science/paper/2QZUVCPJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2107.12460&json=true","fetch_graph":"https://pith.science/api/pith-number/2QZUVCPJ6KWPHI4F32X76QOFNS/graph.json","fetch_events":"https://pith.science/api/pith-number/2QZUVCPJ6KWPHI4F32X76QOFNS/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS/action/timestamp_anchor","attest_storage":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS/action/storage_attestation","attest_author":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS/action/author_attestation","sign_citation":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS/action/citation_signature","submit_replication":"https://pith.science/pith/2QZUVCPJ6KWPHI4F32X76QOFNS/action/replication_record"}},"created_at":"2026-07-05T03:00:56.510741+00:00","updated_at":"2026-07-05T03:00:56.510741+00:00"}