{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:PZDKSFHLTA2Q3GTNIX3H6XPNTI","short_pith_number":"pith:PZDKSFHL","canonical_record":{"source":{"id":"2408.00624","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"eess.AS","submitted_at":"2024-08-01T15:09:32Z","cross_cats_sorted":["cs.CL","cs.CV"],"title_canon_sha256":"335da62c26772c8c8ed4834d1efda066cfc3f109649afd2e1f400cb3ac06cd79","abstract_canon_sha256":"350aa8fb242149d0ccb39aa600c664e205090778e667f929ed03be1af3fb6973"},"schema_version":"1.0"},"canonical_sha256":"7e46a914eb98350d9a6d45f67f5ded9a069c3cffa1445c41a59c46a0ac4231ce","source":{"kind":"arxiv","id":"2408.00624","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.00624","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"arxiv_version","alias_value":"2408.00624v1","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.00624","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"pith_short_12","alias_value":"PZDKSFHLTA2Q","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"pith_short_16","alias_value":"PZDKSFHLTA2Q3GTN","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"pith_short_8","alias_value":"PZDKSFHL","created_at":"2026-07-05T08:51:06Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:PZDKSFHLTA2Q3GTNIX3H6XPNTI","target":"record","payload":{"canonical_record":{"source":{"id":"2408.00624","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"eess.AS","submitted_at":"2024-08-01T15:09:32Z","cross_cats_sorted":["cs.CL","cs.CV"],"title_canon_sha256":"335da62c26772c8c8ed4834d1efda066cfc3f109649afd2e1f400cb3ac06cd79","abstract_canon_sha256":"350aa8fb242149d0ccb39aa600c664e205090778e667f929ed03be1af3fb6973"},"schema_version":"1.0"},"canonical_sha256":"7e46a914eb98350d9a6d45f67f5ded9a069c3cffa1445c41a59c46a0ac4231ce","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:51:06.763339Z","signature_b64":"gpFL3aOmGGRbCT3YRxsJTbyGgg4YJ6moR5NF+grVZVNWy3xIUBJhV/uUWfO/SHTZ4nLqty49MmKcAiua9cCbBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"7e46a914eb98350d9a6d45f67f5ded9a069c3cffa1445c41a59c46a0ac4231ce","last_reissued_at":"2026-07-05T08:51:06.762857Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:51:06.762857Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.00624","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:51:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kF25kMkyyMGDxB6T//XUPKmVeS3HkQ+tIquQ/aF6u2NV0Gx0J6rjZ9aGJT2eGuvN7WlA13gU+9eSL2BpOrmiCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T01:32:28.325268Z"},"content_sha256":"b698867ab70e2bc0d0c57de8c6a3d655e30b1dc98a7ed43e80ce9821bb2740ea","schema_version":"1.0","event_id":"sha256:b698867ab70e2bc0d0c57de8c6a3d655e30b1dc98a7ed43e80ce9821bb2740ea"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:PZDKSFHLTA2Q3GTNIX3H6XPNTI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SynesLM: A Unified Approach for Audio-visual Speech Recognition and Translation via Language Model and Synthetic Data","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.CL","cs.CV"],"primary_cat":"eess.AS","authors_text":"Hengwei Bian, Jiaqi Song, Shinji Watanabe, Soumi Maiti, Xuankai Chang, Yichen Lu","submitted_at":"2024-08-01T15:09:32Z","abstract_excerpt":"In this work, we present SynesLM, an unified model which can perform three multimodal language understanding tasks: audio-visual automatic speech recognition(AV-ASR) and visual-aided speech/machine translation(VST/VMT). Unlike previous research that focused on lip motion as visual cues for speech signals, our work explores more general visual information within entire frames, such as objects and actions. Additionally, we use synthetic image data to enhance the correlation between image and speech data. We benchmark SynesLM against the How2 dataset, demonstrating performance on par with state-o"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.00624","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.00624/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:51:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4QvLgrVOSrEbE57mynfdebkxuZLKdDKG0zG1lQ1n3l0fhj4mSydctZ4gNi5dOln5XScIZ/ObzfW8AgySSMw/CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T01:32:28.326000Z"},"content_sha256":"3a5a27c36e220e9eab69d2bb3ac797acb0ff643292079ae505dedcf2190b0e19","schema_version":"1.0","event_id":"sha256:3a5a27c36e220e9eab69d2bb3ac797acb0ff643292079ae505dedcf2190b0e19"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/PZDKSFHLTA2Q3GTNIX3H6XPNTI/bundle.json","state_url":"https://pith.science/pith/PZDKSFHLTA2Q3GTNIX3H6XPNTI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/PZDKSFHLTA2Q3GTNIX3H6XPNTI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T01:32:28Z","links":{"resolver":"https://pith.science/pith/PZDKSFHLTA2Q3GTNIX3H6XPNTI","bundle":"https://pith.science/pith/PZDKSFHLTA2Q3GTNIX3H6XPNTI/bundle.json","state":"https://pith.science/pith/PZDKSFHLTA2Q3GTNIX3H6XPNTI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/PZDKSFHLTA2Q3GTNIX3H6XPNTI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:PZDKSFHLTA2Q3GTNIX3H6XPNTI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"350aa8fb242149d0ccb39aa600c664e205090778e667f929ed03be1af3fb6973","cross_cats_sorted":["cs.CL","cs.CV"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"eess.AS","submitted_at":"2024-08-01T15:09:32Z","title_canon_sha256":"335da62c26772c8c8ed4834d1efda066cfc3f109649afd2e1f400cb3ac06cd79"},"schema_version":"1.0","source":{"id":"2408.00624","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.00624","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"arxiv_version","alias_value":"2408.00624v1","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.00624","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"pith_short_12","alias_value":"PZDKSFHLTA2Q","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"pith_short_16","alias_value":"PZDKSFHLTA2Q3GTN","created_at":"2026-07-05T08:51:06Z"},{"alias_kind":"pith_short_8","alias_value":"PZDKSFHL","created_at":"2026-07-05T08:51:06Z"}],"graph_snapshots":[{"event_id":"sha256:3a5a27c36e220e9eab69d2bb3ac797acb0ff643292079ae505dedcf2190b0e19","target":"graph","created_at":"2026-07-05T08:51:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.00624/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this work, we present SynesLM, an unified model which can perform three multimodal language understanding tasks: audio-visual automatic speech recognition(AV-ASR) and visual-aided speech/machine translation(VST/VMT). Unlike previous research that focused on lip motion as visual cues for speech signals, our work explores more general visual information within entire frames, such as objects and actions. Additionally, we use synthetic image data to enhance the correlation between image and speech data. We benchmark SynesLM against the How2 dataset, demonstrating performance on par with state-o","authors_text":"Hengwei Bian, Jiaqi Song, Shinji Watanabe, Soumi Maiti, Xuankai Chang, Yichen Lu","cross_cats":["cs.CL","cs.CV"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"eess.AS","submitted_at":"2024-08-01T15:09:32Z","title":"SynesLM: A Unified Approach for Audio-visual Speech Recognition and Translation via Language Model and Synthetic Data"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.00624","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b698867ab70e2bc0d0c57de8c6a3d655e30b1dc98a7ed43e80ce9821bb2740ea","target":"record","created_at":"2026-07-05T08:51:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"350aa8fb242149d0ccb39aa600c664e205090778e667f929ed03be1af3fb6973","cross_cats_sorted":["cs.CL","cs.CV"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"eess.AS","submitted_at":"2024-08-01T15:09:32Z","title_canon_sha256":"335da62c26772c8c8ed4834d1efda066cfc3f109649afd2e1f400cb3ac06cd79"},"schema_version":"1.0","source":{"id":"2408.00624","kind":"arxiv","version":1}},"canonical_sha256":"7e46a914eb98350d9a6d45f67f5ded9a069c3cffa1445c41a59c46a0ac4231ce","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"7e46a914eb98350d9a6d45f67f5ded9a069c3cffa1445c41a59c46a0ac4231ce","first_computed_at":"2026-07-05T08:51:06.762857Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:51:06.762857Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gpFL3aOmGGRbCT3YRxsJTbyGgg4YJ6moR5NF+grVZVNWy3xIUBJhV/uUWfO/SHTZ4nLqty49MmKcAiua9cCbBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:51:06.763339Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.00624","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b698867ab70e2bc0d0c57de8c6a3d655e30b1dc98a7ed43e80ce9821bb2740ea","sha256:3a5a27c36e220e9eab69d2bb3ac797acb0ff643292079ae505dedcf2190b0e19"],"state_sha256":"6a6cd1fa2b573b8d1d4e2796194034e7a71e861f02c8de0f90e2a1216cbf8c30"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eEapjUOSV/+IVo0WioojebOh5H8kM/359Qh72iNLHPKgcwUVwrbJjCBHsJJwBNyjVe7e4KeUCWkSW1d3MKgNAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T01:32:28.331935Z","bundle_sha256":"9151e7be2c04dc9a1997fba9fc20b6826b06361f2fc4065004fa3604d5362a05"}}