{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:J4ZPVPJEBKRVQUWIAYR27T5IR7","short_pith_number":"pith:J4ZPVPJE","canonical_record":{"source":{"id":"2305.18294","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-29T17:59:15Z","cross_cats_sorted":[],"title_canon_sha256":"aaf80119a78baae0f17bdf04d054b0f519220f23953e72b57d820c5065a7d02b","abstract_canon_sha256":"579fe43fa924e80c080bd3125f193c7ab82d51e433cd8d3909d720f50b39aa26"},"schema_version":"1.0"},"canonical_sha256":"4f32fabd240aa35852c80623afcfa88fe9d0b2776b5b8184b0897977c0bebc7e","source":{"kind":"arxiv","id":"2305.18294","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.18294","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"arxiv_version","alias_value":"2305.18294v1","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.18294","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"pith_short_12","alias_value":"J4ZPVPJEBKRV","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"pith_short_16","alias_value":"J4ZPVPJEBKRVQUWI","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"pith_short_8","alias_value":"J4ZPVPJE","created_at":"2026-07-05T06:14:54Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:J4ZPVPJEBKRVQUWIAYR27T5IR7","target":"record","payload":{"canonical_record":{"source":{"id":"2305.18294","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-29T17:59:15Z","cross_cats_sorted":[],"title_canon_sha256":"aaf80119a78baae0f17bdf04d054b0f519220f23953e72b57d820c5065a7d02b","abstract_canon_sha256":"579fe43fa924e80c080bd3125f193c7ab82d51e433cd8d3909d720f50b39aa26"},"schema_version":"1.0"},"canonical_sha256":"4f32fabd240aa35852c80623afcfa88fe9d0b2776b5b8184b0897977c0bebc7e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:14:54.729283Z","signature_b64":"G2iC6vgyv5x5wNBMscXA0HgWq4qX08nrIk8x0WqES3VdTOlVs6EfQRDmVwSwDx9pRk2vRyUHSat/iUdOQiKmAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4f32fabd240aa35852c80623afcfa88fe9d0b2776b5b8184b0897977c0bebc7e","last_reissued_at":"2026-07-05T06:14:54.728839Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:14:54.728839Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2305.18294","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:14:54Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0u2WxpcZNF2A7OWlInu8IqzNWv019d7nRdIphJbhwjMjfR4DGKMN/jWQm4F7/hvTNeIhZm7lD4HhdKww2gKqDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T19:54:06.465342Z"},"content_sha256":"90894633d7fd902252780d9feed0517bffa556e369aac70ca3e27ee1b3f08809","schema_version":"1.0","event_id":"sha256:90894633d7fd902252780d9feed0517bffa556e369aac70ca3e27ee1b3f08809"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:J4ZPVPJEBKRVQUWIAYR27T5IR7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Transformer Language Models Handle Word Frequency in Prediction Head","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Goro Kobayashi, Kentaro Inui, Sho Yokoi, Tatsuki Kuribayashi","submitted_at":"2023-05-29T17:59:15Z","abstract_excerpt":"Prediction head is a crucial component of Transformer language models. Despite its direct impact on prediction, this component has often been overlooked in analyzing Transformers. In this study, we investigate the inner workings of the prediction head, specifically focusing on bias parameters. Our experiments with BERT and GPT-2 models reveal that the biases in their word prediction heads play a significant role in the models' ability to reflect word frequency in a corpus, aligning with the logit adjustment method commonly used in long-tailed learning. We also quantify the effect of controllin"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.18294","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2305.18294/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:14:54Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VGEOff//ptid414g88+9/aXjPnzRqRUzbHxwKDGqSO14X7630z41CMCXBlQtI8RbDZO+EhLTE3gzHxgjiQkEBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T19:54:06.465843Z"},"content_sha256":"0cacffde073d5a87c8a404419029631bc88a087ae274d86c68d7bd7cb3d45011","schema_version":"1.0","event_id":"sha256:0cacffde073d5a87c8a404419029631bc88a087ae274d86c68d7bd7cb3d45011"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/J4ZPVPJEBKRVQUWIAYR27T5IR7/bundle.json","state_url":"https://pith.science/pith/J4ZPVPJEBKRVQUWIAYR27T5IR7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/J4ZPVPJEBKRVQUWIAYR27T5IR7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T19:54:06Z","links":{"resolver":"https://pith.science/pith/J4ZPVPJEBKRVQUWIAYR27T5IR7","bundle":"https://pith.science/pith/J4ZPVPJEBKRVQUWIAYR27T5IR7/bundle.json","state":"https://pith.science/pith/J4ZPVPJEBKRVQUWIAYR27T5IR7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/J4ZPVPJEBKRVQUWIAYR27T5IR7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:J4ZPVPJEBKRVQUWIAYR27T5IR7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"579fe43fa924e80c080bd3125f193c7ab82d51e433cd8d3909d720f50b39aa26","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-29T17:59:15Z","title_canon_sha256":"aaf80119a78baae0f17bdf04d054b0f519220f23953e72b57d820c5065a7d02b"},"schema_version":"1.0","source":{"id":"2305.18294","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2305.18294","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"arxiv_version","alias_value":"2305.18294v1","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2305.18294","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"pith_short_12","alias_value":"J4ZPVPJEBKRV","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"pith_short_16","alias_value":"J4ZPVPJEBKRVQUWI","created_at":"2026-07-05T06:14:54Z"},{"alias_kind":"pith_short_8","alias_value":"J4ZPVPJE","created_at":"2026-07-05T06:14:54Z"}],"graph_snapshots":[{"event_id":"sha256:0cacffde073d5a87c8a404419029631bc88a087ae274d86c68d7bd7cb3d45011","target":"graph","created_at":"2026-07-05T06:14:54Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2305.18294/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Prediction head is a crucial component of Transformer language models. Despite its direct impact on prediction, this component has often been overlooked in analyzing Transformers. In this study, we investigate the inner workings of the prediction head, specifically focusing on bias parameters. Our experiments with BERT and GPT-2 models reveal that the biases in their word prediction heads play a significant role in the models' ability to reflect word frequency in a corpus, aligning with the logit adjustment method commonly used in long-tailed learning. We also quantify the effect of controllin","authors_text":"Goro Kobayashi, Kentaro Inui, Sho Yokoi, Tatsuki Kuribayashi","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-29T17:59:15Z","title":"Transformer Language Models Handle Word Frequency in Prediction Head"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2305.18294","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:90894633d7fd902252780d9feed0517bffa556e369aac70ca3e27ee1b3f08809","target":"record","created_at":"2026-07-05T06:14:54Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"579fe43fa924e80c080bd3125f193c7ab82d51e433cd8d3909d720f50b39aa26","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-05-29T17:59:15Z","title_canon_sha256":"aaf80119a78baae0f17bdf04d054b0f519220f23953e72b57d820c5065a7d02b"},"schema_version":"1.0","source":{"id":"2305.18294","kind":"arxiv","version":1}},"canonical_sha256":"4f32fabd240aa35852c80623afcfa88fe9d0b2776b5b8184b0897977c0bebc7e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4f32fabd240aa35852c80623afcfa88fe9d0b2776b5b8184b0897977c0bebc7e","first_computed_at":"2026-07-05T06:14:54.728839Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:14:54.728839Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"G2iC6vgyv5x5wNBMscXA0HgWq4qX08nrIk8x0WqES3VdTOlVs6EfQRDmVwSwDx9pRk2vRyUHSat/iUdOQiKmAg==","signature_status":"signed_v1","signed_at":"2026-07-05T06:14:54.729283Z","signed_message":"canonical_sha256_bytes"},"source_id":"2305.18294","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:90894633d7fd902252780d9feed0517bffa556e369aac70ca3e27ee1b3f08809","sha256:0cacffde073d5a87c8a404419029631bc88a087ae274d86c68d7bd7cb3d45011"],"state_sha256":"efb0b5618d662e96662e8140b1036409c62459553bffa4a27ba3b5c0a2a0e59b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"d5v9/HotHICuB6EqPTKYVKX9g/MD78rHwC9uyFUC64DgvgGbyPSMB9xc0j8KPPyvkP1AW82TwwwwUlE82VrjDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T19:54:06.469529Z","bundle_sha256":"8b43d61db6a56264cea3eafa5fb7071c09935f88ebce780ea8f6fb613c2c0953"}}