{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2022:64WQACFZVINXWOLBILHMFHEHQL","short_pith_number":"pith:64WQACFZ","canonical_record":{"source":{"id":"2206.07808","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-15T20:44:23Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"b64a935e5c1ede0c12cca1fdead17ad61720b82ae87521297d24d89da7ecc194","abstract_canon_sha256":"49606f1387e70ddaf85703103725452d2d386beea3e6e63a95b62547048b034a"},"schema_version":"1.0"},"canonical_sha256":"f72d0008b9aa1b7b396142cec29c8782ea71aa40a59b5aa6b4662febd1c31d0f","source":{"kind":"arxiv","id":"2206.07808","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.07808","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"arxiv_version","alias_value":"2206.07808v1","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.07808","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"pith_short_12","alias_value":"64WQACFZVINX","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"pith_short_16","alias_value":"64WQACFZVINXWOLB","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"pith_short_8","alias_value":"64WQACFZ","created_at":"2026-07-05T04:32:18Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2022:64WQACFZVINXWOLBILHMFHEHQL","target":"record","payload":{"canonical_record":{"source":{"id":"2206.07808","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-15T20:44:23Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"b64a935e5c1ede0c12cca1fdead17ad61720b82ae87521297d24d89da7ecc194","abstract_canon_sha256":"49606f1387e70ddaf85703103725452d2d386beea3e6e63a95b62547048b034a"},"schema_version":"1.0"},"canonical_sha256":"f72d0008b9aa1b7b396142cec29c8782ea71aa40a59b5aa6b4662febd1c31d0f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T04:32:18.557188Z","signature_b64":"wbzlykkWpTXoKtSyMoG7vnjOV57pjgapU0KsvaaxuSb8rI+QRabh474LcHzay6gugomElcTYrWg/SqsEExTqAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"f72d0008b9aa1b7b396142cec29c8782ea71aa40a59b5aa6b4662febd1c31d0f","last_reissued_at":"2026-07-05T04:32:18.556753Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T04:32:18.556753Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2206.07808","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:32:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xERdyE6ANZykxl+epxtoPJZmT+mnzdDsAaVUrISiXt1F345ef/X/bv5yonuDY7HKJ+bjvOGz7Ye5d2gcH4c5Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T13:30:54.797853Z"},"content_sha256":"a77199c1b13e7454b3a33d1c197c197fbfc9dfd1afd4544d1efcf57c22c94984","schema_version":"1.0","event_id":"sha256:a77199c1b13e7454b3a33d1c197c197fbfc9dfd1afd4544d1efcf57c22c94984"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2022:64WQACFZVINXWOLBILHMFHEHQL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Alexa Teacher Model: Pretraining and Distilling Multi-Billion-Parameter Encoders for Natural Language Understanding Systems","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Abhishek Bhagia, Alessandro Manzotti, Amit Chauhan, Andy Rosenbaum, Anjali Shenoy, Anurag Dwarakanath, Beiye Liu, Chandana Satya Prakash, Charith Peris, Claudio Delli Bovi, Davide Bernardi, Dilek Hakkani-Tur, Enrico Palumbo, Fabian Triefenbach, Gokhan Tur, Gokmen Oz, Haidar Khan, Haiyang Yu, Jack FitzGerald, Jianhua Lu, Jin Cao, Jonathan Hueser, Karolina Owczarzak, Karthik Gopalakrishnan, Kevin Martin Jose, Konstantine Arkoudas, Liz Tan, Luoxin Chen, Mukund Harakere Sridhar, Pan Wei, Pradeep Natarajan, Prem Natarajan, Rakesh Chada, Saleh Soltan, Satyam Dwivedi, Shankar Ananthakrishnan, Shuai Zheng, Stephen Rawls, Thomas Gueudre, Turan Gojayev, Wael Hamza","submitted_at":"2022-06-15T20:44:23Z","abstract_excerpt":"We present results from a large-scale experiment on pretraining encoders with non-embedding parameter counts ranging from 700M to 9.3B, their subsequent distillation into smaller models ranging from 17M-170M parameters, and their application to the Natural Language Understanding (NLU) component of a virtual assistant system. Though we train using 70% spoken-form data, our teacher models perform comparably to XLM-R and mT5 when evaluated on the written-form Cross-lingual Natural Language Inference (XNLI) corpus. We perform a second stage of pretraining on our teacher models using in-domain data"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2206.07808","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2206.07808/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T04:32:18Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"h9yd4env+7GRpZF6eeLPDKKCr/VcdAg9CPc6rI7Ei6UFQfUi+HmTSrjf+sJzl1DaJH+jAhMJPgMChK4fk3lOAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T13:30:54.798237Z"},"content_sha256":"bb53264a5b76ee4fa8fb2bbed23266a4dd08d1a06d25a48167dbc372b4e7209a","schema_version":"1.0","event_id":"sha256:bb53264a5b76ee4fa8fb2bbed23266a4dd08d1a06d25a48167dbc372b4e7209a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/64WQACFZVINXWOLBILHMFHEHQL/bundle.json","state_url":"https://pith.science/pith/64WQACFZVINXWOLBILHMFHEHQL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/64WQACFZVINXWOLBILHMFHEHQL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T13:30:54Z","links":{"resolver":"https://pith.science/pith/64WQACFZVINXWOLBILHMFHEHQL","bundle":"https://pith.science/pith/64WQACFZVINXWOLBILHMFHEHQL/bundle.json","state":"https://pith.science/pith/64WQACFZVINXWOLBILHMFHEHQL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/64WQACFZVINXWOLBILHMFHEHQL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2022:64WQACFZVINXWOLBILHMFHEHQL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"49606f1387e70ddaf85703103725452d2d386beea3e6e63a95b62547048b034a","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-15T20:44:23Z","title_canon_sha256":"b64a935e5c1ede0c12cca1fdead17ad61720b82ae87521297d24d89da7ecc194"},"schema_version":"1.0","source":{"id":"2206.07808","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2206.07808","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"arxiv_version","alias_value":"2206.07808v1","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2206.07808","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"pith_short_12","alias_value":"64WQACFZVINX","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"pith_short_16","alias_value":"64WQACFZVINXWOLB","created_at":"2026-07-05T04:32:18Z"},{"alias_kind":"pith_short_8","alias_value":"64WQACFZ","created_at":"2026-07-05T04:32:18Z"}],"graph_snapshots":[{"event_id":"sha256:bb53264a5b76ee4fa8fb2bbed23266a4dd08d1a06d25a48167dbc372b4e7209a","target":"graph","created_at":"2026-07-05T04:32:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2206.07808/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"We present results from a large-scale experiment on pretraining encoders with non-embedding parameter counts ranging from 700M to 9.3B, their subsequent distillation into smaller models ranging from 17M-170M parameters, and their application to the Natural Language Understanding (NLU) component of a virtual assistant system. Though we train using 70% spoken-form data, our teacher models perform comparably to XLM-R and mT5 when evaluated on the written-form Cross-lingual Natural Language Inference (XNLI) corpus. We perform a second stage of pretraining on our teacher models using in-domain data","authors_text":"Abhishek Bhagia, Alessandro Manzotti, Amit Chauhan, Andy Rosenbaum, Anjali Shenoy, Anurag Dwarakanath, Beiye Liu, Chandana Satya Prakash, Charith Peris, Claudio Delli Bovi, Davide Bernardi, Dilek Hakkani-Tur, Enrico Palumbo, Fabian Triefenbach, Gokhan Tur, Gokmen Oz, Haidar Khan, Haiyang Yu, Jack FitzGerald, Jianhua Lu, Jin Cao, Jonathan Hueser, Karolina Owczarzak, Karthik Gopalakrishnan, Kevin Martin Jose, Konstantine Arkoudas, Liz Tan, Luoxin Chen, Mukund Harakere Sridhar, Pan Wei, Pradeep Natarajan, Prem Natarajan, Rakesh Chada, Saleh Soltan, Satyam Dwivedi, Shankar Ananthakrishnan, Shuai Zheng, Stephen Rawls, Thomas Gueudre, Turan Gojayev, Wael Hamza","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-15T20:44:23Z","title":"Alexa Teacher Model: Pretraining and Distilling Multi-Billion-Parameter Encoders for Natural Language Understanding Systems"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2206.07808","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a77199c1b13e7454b3a33d1c197c197fbfc9dfd1afd4544d1efcf57c22c94984","target":"record","created_at":"2026-07-05T04:32:18Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"49606f1387e70ddaf85703103725452d2d386beea3e6e63a95b62547048b034a","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2022-06-15T20:44:23Z","title_canon_sha256":"b64a935e5c1ede0c12cca1fdead17ad61720b82ae87521297d24d89da7ecc194"},"schema_version":"1.0","source":{"id":"2206.07808","kind":"arxiv","version":1}},"canonical_sha256":"f72d0008b9aa1b7b396142cec29c8782ea71aa40a59b5aa6b4662febd1c31d0f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"f72d0008b9aa1b7b396142cec29c8782ea71aa40a59b5aa6b4662febd1c31d0f","first_computed_at":"2026-07-05T04:32:18.556753Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T04:32:18.556753Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"wbzlykkWpTXoKtSyMoG7vnjOV57pjgapU0KsvaaxuSb8rI+QRabh474LcHzay6gugomElcTYrWg/SqsEExTqAw==","signature_status":"signed_v1","signed_at":"2026-07-05T04:32:18.557188Z","signed_message":"canonical_sha256_bytes"},"source_id":"2206.07808","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a77199c1b13e7454b3a33d1c197c197fbfc9dfd1afd4544d1efcf57c22c94984","sha256:bb53264a5b76ee4fa8fb2bbed23266a4dd08d1a06d25a48167dbc372b4e7209a"],"state_sha256":"0f43405288ecfa685a11ebca48827f6113ce63fc757237c4a7dc84cb14a0805a"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"h8Vzy8S7vkRTDH+5xqN3/gD+D7Gimf2CKzBrVGZEAbpUqCiFMmsJzIAUhAS/4mY7UnAmDuRhRT591E+YUrwSAQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T13:30:54.800164Z","bundle_sha256":"e8748a76efedf3e2576f027bc543aa0db595205a09cccf80a760f4d97bf95029"}}