{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:KP7HOAIAIAYWJOJPCYWGQ5QYBP","short_pith_number":"pith:KP7HOAIA","schema_version":"1.0","canonical_sha256":"53fe770100403164b92f162c6876180be24b7925eb83e4b4d39b9a0d349fe3a3","source":{"kind":"arxiv","id":"2406.03706","version":1},"attestation_state":"computed","paper":{"title":"Improving Audio Codec-based Zero-Shot Text-to-Speech Synthesis with Multi-Modal Context and Large Language Model","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","eess.AS"],"primary_cat":"cs.SD","authors_text":"Jinlong Xue, Ya Li, Yayue Deng, Yicheng Han, Yingming Gao","submitted_at":"2024-06-06T03:06:45Z","abstract_excerpt":"Recent advances in large language models (LLMs) and development of audio codecs greatly propel the zero-shot TTS. They can synthesize personalized speech with only a 3-second speech of an unseen speaker as acoustic prompt. However, they only support short speech prompts and cannot leverage longer context information, as required in audiobook and conversational TTS scenarios. In this paper, we introduce a novel audio codec-based TTS model to adapt context features with multiple enhancements. Inspired by the success of Qformer, we propose a multi-modal context-enhanced Qformer (MMCE-Qformer) to "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.03706","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SD","submitted_at":"2024-06-06T03:06:45Z","cross_cats_sorted":["cs.CL","eess.AS"],"title_canon_sha256":"7d7d97376556e4aa831020b9b46e47d2754491256263686b2758f7d10aa0aa48","abstract_canon_sha256":"ea9953f659f5ff717b55898a41e8b53242e8c4441a831813178cea3c1b46b46e"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:28:14.949306Z","signature_b64":"sH+VpvYTHSEioV/n21CSBap8jYSzuTIMi3s9JzfmgiaDAwJ4AImX4G8VsqcTdjX2yx22S/y3u264I+Mki2ppCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"53fe770100403164b92f162c6876180be24b7925eb83e4b4d39b9a0d349fe3a3","last_reissued_at":"2026-07-05T08:28:14.948833Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:28:14.948833Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Improving Audio Codec-based Zero-Shot Text-to-Speech Synthesis with Multi-Modal Context and Large Language Model","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","eess.AS"],"primary_cat":"cs.SD","authors_text":"Jinlong Xue, Ya Li, Yayue Deng, Yicheng Han, Yingming Gao","submitted_at":"2024-06-06T03:06:45Z","abstract_excerpt":"Recent advances in large language models (LLMs) and development of audio codecs greatly propel the zero-shot TTS. They can synthesize personalized speech with only a 3-second speech of an unseen speaker as acoustic prompt. However, they only support short speech prompts and cannot leverage longer context information, as required in audiobook and conversational TTS scenarios. In this paper, we introduce a novel audio codec-based TTS model to adapt context features with multiple enhancements. Inspired by the success of Qformer, we propose a multi-modal context-enhanced Qformer (MMCE-Qformer) to "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.03706","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.03706/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.03706","created_at":"2026-07-05T08:28:14.948892+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.03706v1","created_at":"2026-07-05T08:28:14.948892+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.03706","created_at":"2026-07-05T08:28:14.948892+00:00"},{"alias_kind":"pith_short_12","alias_value":"KP7HOAIAIAYW","created_at":"2026-07-05T08:28:14.948892+00:00"},{"alias_kind":"pith_short_16","alias_value":"KP7HOAIAIAYWJOJP","created_at":"2026-07-05T08:28:14.948892+00:00"},{"alias_kind":"pith_short_8","alias_value":"KP7HOAIA","created_at":"2026-07-05T08:28:14.948892+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2604.24794","citing_title":"V.O.I.C.E (Voice, Ownership, Identity, Control, Expression): Risk Taxonomy of Synthetic Voice Generation From Empirical Data","ref_index":88,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP","json":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP.json","graph_json":"https://pith.science/api/pith-number/KP7HOAIAIAYWJOJPCYWGQ5QYBP/graph.json","events_json":"https://pith.science/api/pith-number/KP7HOAIAIAYWJOJPCYWGQ5QYBP/events.json","paper":"https://pith.science/paper/KP7HOAIA"},"agent_actions":{"view_html":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP","download_json":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP.json","view_paper":"https://pith.science/paper/KP7HOAIA","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.03706&json=true","fetch_graph":"https://pith.science/api/pith-number/KP7HOAIAIAYWJOJPCYWGQ5QYBP/graph.json","fetch_events":"https://pith.science/api/pith-number/KP7HOAIAIAYWJOJPCYWGQ5QYBP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP/action/storage_attestation","attest_author":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP/action/author_attestation","sign_citation":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP/action/citation_signature","submit_replication":"https://pith.science/pith/KP7HOAIAIAYWJOJPCYWGQ5QYBP/action/replication_record"}},"created_at":"2026-07-05T08:28:14.948892+00:00","updated_at":"2026-07-05T08:28:14.948892+00:00"}