{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:M5PNVHOTEQBSTKJOJFALHSYJXH","short_pith_number":"pith:M5PNVHOT","schema_version":"1.0","canonical_sha256":"675eda9dd3240329a92e4940b3cb09b9c9e316a93e0270fad8a07cb60a637340","source":{"kind":"arxiv","id":"2407.04051","version":3},"attestation_state":"computed","paper":{"title":"FunAudioLLM: Voice Understanding and Generation Foundation Models for Natural Interaction Between Humans and LLMs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","eess.AS"],"primary_cat":"cs.SD","authors_text":"Bin Ma, Bin Zhang, Changfeng Gao, Changhe Song, Chong Deng, Chongjia Ni, Hangrui Hu, Haoneng Luo, Hao Wang, Heng Lu, Jiaqi Shi, Kai Hu, Keyu An, Nan Zhao, Qian Chen, Qinglin Zhang, Shengpeng Ji, Shiliang Zhang, Siqi Zheng, Ting He, Wen Wang, Xiang Lv, Xian Shi, Yabin Li, Yexin Yang, Yue Gu, Yuxuan Wang, Zerui Li, Zhangyu Xiao, Zhifu Gao, Zhihao Du, Zhijie Yan, Ziyang Ma","submitted_at":"2024-07-04T16:49:02Z","abstract_excerpt":"This report introduces FunAudioLLM, a model family designed to enhance natural voice interactions between humans and large language models (LLMs). At its core are two innovative models: SenseVoice, which handles multilingual speech recognition, emotion recognition, and audio event detection; and CosyVoice, which facilitates natural speech generation with control over multiple languages, timbre, speaking style, and speaker identity. SenseVoice-Small delivers exceptionally low-latency ASR for 5 languages, and SenseVoice-Large supports high-precision ASR for over 50 languages, while CosyVoice exc"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2407.04051","kind":"arxiv","version":3},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.SD","submitted_at":"2024-07-04T16:49:02Z","cross_cats_sorted":["cs.AI","eess.AS"],"title_canon_sha256":"12a9b1b0f0e56d65b62e51c57a72faba69b2f63847cabbf6eebe3abcc93afe71","abstract_canon_sha256":"6d65ee4981ce17d499fe6a6b6b791b8581ea73f9df9692bc8597236ffe9d35d7"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:42:38.299200Z","signature_b64":"xNjyKFMvFpprOQ2ijq6F6pJT87T5siGGVWOddRIE/Pcpo5KIYwkzQ0JvEBftXQOjTM5ci5Jxbq1wCKRDbgCZAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"675eda9dd3240329a92e4940b3cb09b9c9e316a93e0270fad8a07cb60a637340","last_reissued_at":"2026-07-05T08:42:38.298698Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:42:38.298698Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"FunAudioLLM: Voice Understanding and Generation Foundation Models for Natural Interaction Between Humans and LLMs","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","eess.AS"],"primary_cat":"cs.SD","authors_text":"Bin Ma, Bin Zhang, Changfeng Gao, Changhe Song, Chong Deng, Chongjia Ni, Hangrui Hu, Haoneng Luo, Hao Wang, Heng Lu, Jiaqi Shi, Kai Hu, Keyu An, Nan Zhao, Qian Chen, Qinglin Zhang, Shengpeng Ji, Shiliang Zhang, Siqi Zheng, Ting He, Wen Wang, Xiang Lv, Xian Shi, Yabin Li, Yexin Yang, Yue Gu, Yuxuan Wang, Zerui Li, Zhangyu Xiao, Zhifu Gao, Zhihao Du, Zhijie Yan, Ziyang Ma","submitted_at":"2024-07-04T16:49:02Z","abstract_excerpt":"This report introduces FunAudioLLM, a model family designed to enhance natural voice interactions between humans and large language models (LLMs). At its core are two innovative models: SenseVoice, which handles multilingual speech recognition, emotion recognition, and audio event detection; and CosyVoice, which facilitates natural speech generation with control over multiple languages, timbre, speaking style, and speaker identity. SenseVoice-Small delivers exceptionally low-latency ASR for 5 languages, and SenseVoice-Large supports high-precision ASR for over 50 languages, while CosyVoice exc"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.04051","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.04051/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2407.04051","created_at":"2026-07-05T08:42:38.298766+00:00"},{"alias_kind":"arxiv_version","alias_value":"2407.04051v3","created_at":"2026-07-05T08:42:38.298766+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.04051","created_at":"2026-07-05T08:42:38.298766+00:00"},{"alias_kind":"pith_short_12","alias_value":"M5PNVHOTEQBS","created_at":"2026-07-05T08:42:38.298766+00:00"},{"alias_kind":"pith_short_16","alias_value":"M5PNVHOTEQBSTKJO","created_at":"2026-07-05T08:42:38.298766+00:00"},{"alias_kind":"pith_short_8","alias_value":"M5PNVHOT","created_at":"2026-07-05T08:42:38.298766+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":28,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2604.18326","citing_title":"OmniHuman: A Large-scale Dataset and Benchmark for Human-Centric Video Generation","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2606.22276","citing_title":"Learning from Audio-Dependency Errors: Data Curation Strategies Based on Model Confusion Patterns in Audio Question Answering","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2606.13095","citing_title":"Balancing ASR and diarization in end-to-end LLMs for multi-talker speech recognition","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2605.30993","citing_title":"SwanVoice: Expressive Long-Form Zero-Shot Speech Synthesis for Both Monologue and Dialogue","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2606.31247","citing_title":"FlexiSLM: A Dynamic and Controllable Frame Rate Spoken Language Model","ref_index":100,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19266","citing_title":"FormalASR: End-to-End Spoken Chinese to Formal Text","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2606.01016","citing_title":"PolySpeech-100: A Large-Scale Benchmark for Speech Understanding Across 100+ Languages and Dialects","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2504.08528","citing_title":"On The Landscape of Spoken Language Models: A Comprehensive Survey","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2512.01537","citing_title":"Two-Dimensional Quantization for Geometry-Aware Audio Coding","ref_index":68,"is_internal_anchor":false},{"citing_arxiv_id":"2605.20266","citing_title":"A Survey of Large Audio Language Models: Generalization, Trustworthiness, and Outlook","ref_index":115,"is_internal_anchor":false},{"citing_arxiv_id":"2605.19266","citing_title":"FormalASR: End-to-End Spoken Chinese to Formal Text","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2509.24708","citing_title":"SenSE: Semantic-Aware High-Fidelity Universal Speech Enhancement","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2412.02612","citing_title":"GLM-4-Voice: Towards Intelligent and Human-Like End-to-End Spoken Chatbot","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2412.02612","citing_title":"GLM-4-Voice: Towards Intelligent and Human-Like End-to-End Spoken Chatbot","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2602.12783","citing_title":"SQuTR: A Robustness Benchmark for Spoken Query to Text Retrieval under Acoustic Noise","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2603.05094","citing_title":"TW-Sound580K: A Regional Audio-Text Dataset with Verification-Guided Curation for Localized Audio-Language Modeling","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2603.09677","citing_title":"Logics-Parsing-Omni Technical Report","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2603.21664","citing_title":"HumanOmni-Speaker: Identifying Who said What and When","ref_index":38,"is_internal_anchor":false},{"citing_arxiv_id":"2604.00688","citing_title":"OmniVoice: Towards Omnilingual Zero-Shot Text-to-Speech with Diffusion Language Models","ref_index":60,"is_internal_anchor":false},{"citing_arxiv_id":"2604.25819","citing_title":"Mutual Forcing: Dual-Mode Self-Evolution for Fast Autoregressive Audio-Video Character Generation","ref_index":2,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04613","citing_title":"VocalParse: Towards Unified and Scalable Singing Voice Transcription with Large Audio Language Models","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.03937","citing_title":"MiniMind-O Technical Report: An Open Small-Scale Speech-Native Omni Model","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2604.08384","citing_title":"TASU2: Controllable CTC Simulation for Alignment and Low-Resource Adaptation of Speech LLMs","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2604.08000","citing_title":"PASK: Toward Intent-Aware Proactive Agents with Long-Term Memory","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06765","citing_title":"VITA-QinYu: Expressive Spoken Language Model for Role-Playing and Singing","ref_index":18,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH","json":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH.json","graph_json":"https://pith.science/api/pith-number/M5PNVHOTEQBSTKJOJFALHSYJXH/graph.json","events_json":"https://pith.science/api/pith-number/M5PNVHOTEQBSTKJOJFALHSYJXH/events.json","paper":"https://pith.science/paper/M5PNVHOT"},"agent_actions":{"view_html":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH","download_json":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH.json","view_paper":"https://pith.science/paper/M5PNVHOT","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2407.04051&json=true","fetch_graph":"https://pith.science/api/pith-number/M5PNVHOTEQBSTKJOJFALHSYJXH/graph.json","fetch_events":"https://pith.science/api/pith-number/M5PNVHOTEQBSTKJOJFALHSYJXH/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH/action/timestamp_anchor","attest_storage":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH/action/storage_attestation","attest_author":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH/action/author_attestation","sign_citation":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH/action/citation_signature","submit_replication":"https://pith.science/pith/M5PNVHOTEQBSTKJOJFALHSYJXH/action/replication_record"}},"created_at":"2026-07-05T08:42:38.298766+00:00","updated_at":"2026-07-05T08:42:38.298766+00:00"}