{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:TI3QM4BINPABMXLQFBH3YLU2B5","short_pith_number":"pith:TI3QM4BI","canonical_record":{"source":{"id":"2105.06020","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-13T01:10:51Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"831844e8b34460417769ea0169276f2f451d55fe4a73f3a00eb44c37a6833a83","abstract_canon_sha256":"a1d0421ea779da0455cee79e3801789a441aace55872cbcf0b0c7c8114709a11"},"schema_version":"1.0"},"canonical_sha256":"9a370670286bc0165d70284fbc2e9a0f4996581ba46f5e0e1e552bf0d1ad44b2","source":{"kind":"arxiv","id":"2105.06020","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.06020","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"arxiv_version","alias_value":"2105.06020v1","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.06020","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"pith_short_12","alias_value":"TI3QM4BINPAB","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"pith_short_16","alias_value":"TI3QM4BINPABMXLQ","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"pith_short_8","alias_value":"TI3QM4BI","created_at":"2026-07-05T02:40:03Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:TI3QM4BINPABMXLQFBH3YLU2B5","target":"record","payload":{"canonical_record":{"source":{"id":"2105.06020","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-13T01:10:51Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"831844e8b34460417769ea0169276f2f451d55fe4a73f3a00eb44c37a6833a83","abstract_canon_sha256":"a1d0421ea779da0455cee79e3801789a441aace55872cbcf0b0c7c8114709a11"},"schema_version":"1.0"},"canonical_sha256":"9a370670286bc0165d70284fbc2e9a0f4996581ba46f5e0e1e552bf0d1ad44b2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T02:40:03.359301Z","signature_b64":"mZqqfSiIShfmeL2FyoKNdRQco0jc1YdC3yRRrGb9fXxYpiRI2HvlzLLhS+I2Rdd3LP7XSM0sHvJ0rM3CUbenDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"9a370670286bc0165d70284fbc2e9a0f4996581ba46f5e0e1e552bf0d1ad44b2","last_reissued_at":"2026-07-05T02:40:03.358959Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T02:40:03.358959Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2105.06020","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:40:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3lRhvEsnR5C6+1BTzKT6sJcEvS5TJIPDNhfX79PXSLDJgMK8dEqhaVcyLbH/Ej/oWM9s1UoQzxSSIpCDTfdMDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T03:57:00.843438Z"},"content_sha256":"02d1779f03d7c481e8444351ac7ab1e2761af2da1cd22a706ab4727572b0b57d","schema_version":"1.0","event_id":"sha256:02d1779f03d7c481e8444351ac7ab1e2761af2da1cd22a706ab4727572b0b57d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:TI3QM4BINPABMXLQFBH3YLU2B5","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Are Larger Pretrained Language Models Uniformly Better? Comparing Performance at the Instance Level","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Dan Klein, Dhruba Ghosh, Jacob Steinhardt, Ruiqi Zhong","submitted_at":"2021-05-13T01:10:51Z","abstract_excerpt":"Larger language models have higher accuracy on average, but are they better on every single instance (datapoint)? Some work suggests larger models have higher out-of-distribution robustness, while other work suggests they have lower accuracy on rare subgroups. To understand these differences, we investigate these models at the level of individual instances. However, one major challenge is that individual predictions are highly sensitive to noise in the randomness in training. We develop statistically rigorous methods to address this, and after accounting for pretraining and finetuning noise, w"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.06020","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2105.06020/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T02:40:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"EZ0TXkLEnmUhFoTTfj23Zl7pNg5AmJqHQdSY+dKvz56s7KnYo5cbdwaT4MovI4ZP76utQf0MH9pe8spn7LAaDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-07T03:57:00.849744Z"},"content_sha256":"64279ca9e3f5516315e56fdd047bf3c95dca702eee1da8383330aceec7cc945f","schema_version":"1.0","event_id":"sha256:64279ca9e3f5516315e56fdd047bf3c95dca702eee1da8383330aceec7cc945f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/TI3QM4BINPABMXLQFBH3YLU2B5/bundle.json","state_url":"https://pith.science/pith/TI3QM4BINPABMXLQFBH3YLU2B5/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/TI3QM4BINPABMXLQFBH3YLU2B5/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-07T03:57:00Z","links":{"resolver":"https://pith.science/pith/TI3QM4BINPABMXLQFBH3YLU2B5","bundle":"https://pith.science/pith/TI3QM4BINPABMXLQFBH3YLU2B5/bundle.json","state":"https://pith.science/pith/TI3QM4BINPABMXLQFBH3YLU2B5/state.json","well_known_bundle":"https://pith.science/.well-known/pith/TI3QM4BINPABMXLQFBH3YLU2B5/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:TI3QM4BINPABMXLQFBH3YLU2B5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a1d0421ea779da0455cee79e3801789a441aace55872cbcf0b0c7c8114709a11","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-13T01:10:51Z","title_canon_sha256":"831844e8b34460417769ea0169276f2f451d55fe4a73f3a00eb44c37a6833a83"},"schema_version":"1.0","source":{"id":"2105.06020","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2105.06020","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"arxiv_version","alias_value":"2105.06020v1","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2105.06020","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"pith_short_12","alias_value":"TI3QM4BINPAB","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"pith_short_16","alias_value":"TI3QM4BINPABMXLQ","created_at":"2026-07-05T02:40:03Z"},{"alias_kind":"pith_short_8","alias_value":"TI3QM4BI","created_at":"2026-07-05T02:40:03Z"}],"graph_snapshots":[{"event_id":"sha256:64279ca9e3f5516315e56fdd047bf3c95dca702eee1da8383330aceec7cc945f","target":"graph","created_at":"2026-07-05T02:40:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2105.06020/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Larger language models have higher accuracy on average, but are they better on every single instance (datapoint)? Some work suggests larger models have higher out-of-distribution robustness, while other work suggests they have lower accuracy on rare subgroups. To understand these differences, we investigate these models at the level of individual instances. However, one major challenge is that individual predictions are highly sensitive to noise in the randomness in training. We develop statistically rigorous methods to address this, and after accounting for pretraining and finetuning noise, w","authors_text":"Dan Klein, Dhruba Ghosh, Jacob Steinhardt, Ruiqi Zhong","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-13T01:10:51Z","title":"Are Larger Pretrained Language Models Uniformly Better? Comparing Performance at the Instance Level"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2105.06020","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:02d1779f03d7c481e8444351ac7ab1e2761af2da1cd22a706ab4727572b0b57d","target":"record","created_at":"2026-07-05T02:40:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a1d0421ea779da0455cee79e3801789a441aace55872cbcf0b0c7c8114709a11","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-05-13T01:10:51Z","title_canon_sha256":"831844e8b34460417769ea0169276f2f451d55fe4a73f3a00eb44c37a6833a83"},"schema_version":"1.0","source":{"id":"2105.06020","kind":"arxiv","version":1}},"canonical_sha256":"9a370670286bc0165d70284fbc2e9a0f4996581ba46f5e0e1e552bf0d1ad44b2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"9a370670286bc0165d70284fbc2e9a0f4996581ba46f5e0e1e552bf0d1ad44b2","first_computed_at":"2026-07-05T02:40:03.358959Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T02:40:03.358959Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"mZqqfSiIShfmeL2FyoKNdRQco0jc1YdC3yRRrGb9fXxYpiRI2HvlzLLhS+I2Rdd3LP7XSM0sHvJ0rM3CUbenDw==","signature_status":"signed_v1","signed_at":"2026-07-05T02:40:03.359301Z","signed_message":"canonical_sha256_bytes"},"source_id":"2105.06020","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:02d1779f03d7c481e8444351ac7ab1e2761af2da1cd22a706ab4727572b0b57d","sha256:64279ca9e3f5516315e56fdd047bf3c95dca702eee1da8383330aceec7cc945f"],"state_sha256":"3e8efc176e56140abfd6aaa24fa6855dc1cc4bb5e9bbdb26149c98cd1d6327e8"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8oMnlHOBuSaAKaT58vtuuWGqZ+UAv8HqGIqswVy84SJ/RybOQUQiCA3u7JHFX/6waRH+wVqsRlO5Y99SwLJdBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-07T03:57:00.861214Z","bundle_sha256":"bd339c7ecdd3aa986fc2d52bac15e0d1304809cc2795467bd2644ae80e1b944d"}}