{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:ZSDZB6GGVRLMY7CZV3SJJKIJOV","short_pith_number":"pith:ZSDZB6GG","canonical_record":{"source":{"id":"2306.17842","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-06-30T17:59:07Z","cross_cats_sorted":["cs.CL","cs.MM"],"title_canon_sha256":"ef98d20ca95e25fe3df86618ed7140094d082d1b1eff43a060917dabdae0448f","abstract_canon_sha256":"a80664fbdc1bc2a64c5056df5ee581be6a2586cdd3468553d92c6acbcf0a2651"},"schema_version":"1.0"},"canonical_sha256":"cc8790f8c6ac56cc7c59aee494a909756a2c80286407cd11082ae945a0e35d34","source":{"kind":"arxiv","id":"2306.17842","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.17842","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"arxiv_version","alias_value":"2306.17842v3","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.17842","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"pith_short_12","alias_value":"ZSDZB6GGVRLM","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"pith_short_16","alias_value":"ZSDZB6GGVRLMY7CZ","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"pith_short_8","alias_value":"ZSDZB6GG","created_at":"2026-07-05T07:06:31Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:ZSDZB6GGVRLMY7CZV3SJJKIJOV","target":"record","payload":{"canonical_record":{"source":{"id":"2306.17842","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-06-30T17:59:07Z","cross_cats_sorted":["cs.CL","cs.MM"],"title_canon_sha256":"ef98d20ca95e25fe3df86618ed7140094d082d1b1eff43a060917dabdae0448f","abstract_canon_sha256":"a80664fbdc1bc2a64c5056df5ee581be6a2586cdd3468553d92c6acbcf0a2651"},"schema_version":"1.0"},"canonical_sha256":"cc8790f8c6ac56cc7c59aee494a909756a2c80286407cd11082ae945a0e35d34","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:06:31.109818Z","signature_b64":"xMhYeZZh5X5GVOlH/BiFN+oBKfPCdhodRW0VW3H/z+hC3opmukrXO5wxi+MZyQvANzwnWmC4295hLO9wz5D7Dw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"cc8790f8c6ac56cc7c59aee494a909756a2c80286407cd11082ae945a0e35d34","last_reissued_at":"2026-07-05T07:06:31.109303Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:06:31.109303Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2306.17842","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:06:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kC1V5IfWVoaK/3JUz2sddxpzM3o/shsMmimlIy9o94f/QT/lg+0ShLz93TthrVwVzFMcofLyPipK1Y4l8QIfCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:06:22.339024Z"},"content_sha256":"b2e84b42d78cb54ea56eacf5639e5dbef8ef4bc075032dd26db8b8ef48d6ba94","schema_version":"1.0","event_id":"sha256:b2e84b42d78cb54ea56eacf5639e5dbef8ef4bc075032dd26db8b8ef48d6ba94"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:ZSDZB6GGVRLMY7CZV3SJJKIJOV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SPAE: Semantic Pyramid AutoEncoder for Multimodal Generation with Frozen LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.MM"],"primary_cat":"cs.CV","authors_text":"Alexander G. Hauptmann, David A. Ross, Irfan Essa, Kevin Murphy, Lijun Yu, Lu Jiang, Ming-Hsuan Yang, Vivek Kumar, Wolfgang Macherey, Yanping Huang, Yonatan Bisk, Yong Cheng, Zhiruo Wang","submitted_at":"2023-06-30T17:59:07Z","abstract_excerpt":"In this work, we introduce Semantic Pyramid AutoEncoder (SPAE) for enabling frozen LLMs to perform both understanding and generation tasks involving non-linguistic modalities such as images or videos. SPAE converts between raw pixels and interpretable lexical tokens (or words) extracted from the LLM's vocabulary. The resulting tokens capture both the semantic meaning and the fine-grained details needed for visual reconstruction, effectively translating the visual content into a language comprehensible to the LLM, and empowering it to perform a wide array of multimodal tasks. Our approach is va"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.17842","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2306.17842/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:06:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vaGUozAU8I23K3lS7Ag2MCECDnLxni+YYbiQfm+sBe1CgtqqJ9B278O5OxVQ2HdVtbsjT4Mp4w8w/G/rusqcAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T13:06:22.339425Z"},"content_sha256":"69ce12c89007bbaeff0ee691363c3bd9d46faa26ce144e658365a5065b35817a","schema_version":"1.0","event_id":"sha256:69ce12c89007bbaeff0ee691363c3bd9d46faa26ce144e658365a5065b35817a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ZSDZB6GGVRLMY7CZV3SJJKIJOV/bundle.json","state_url":"https://pith.science/pith/ZSDZB6GGVRLMY7CZV3SJJKIJOV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ZSDZB6GGVRLMY7CZV3SJJKIJOV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T13:06:22Z","links":{"resolver":"https://pith.science/pith/ZSDZB6GGVRLMY7CZV3SJJKIJOV","bundle":"https://pith.science/pith/ZSDZB6GGVRLMY7CZV3SJJKIJOV/bundle.json","state":"https://pith.science/pith/ZSDZB6GGVRLMY7CZV3SJJKIJOV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ZSDZB6GGVRLMY7CZV3SJJKIJOV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:ZSDZB6GGVRLMY7CZV3SJJKIJOV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a80664fbdc1bc2a64c5056df5ee581be6a2586cdd3468553d92c6acbcf0a2651","cross_cats_sorted":["cs.CL","cs.MM"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-06-30T17:59:07Z","title_canon_sha256":"ef98d20ca95e25fe3df86618ed7140094d082d1b1eff43a060917dabdae0448f"},"schema_version":"1.0","source":{"id":"2306.17842","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2306.17842","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"arxiv_version","alias_value":"2306.17842v3","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2306.17842","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"pith_short_12","alias_value":"ZSDZB6GGVRLM","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"pith_short_16","alias_value":"ZSDZB6GGVRLMY7CZ","created_at":"2026-07-05T07:06:31Z"},{"alias_kind":"pith_short_8","alias_value":"ZSDZB6GG","created_at":"2026-07-05T07:06:31Z"}],"graph_snapshots":[{"event_id":"sha256:69ce12c89007bbaeff0ee691363c3bd9d46faa26ce144e658365a5065b35817a","target":"graph","created_at":"2026-07-05T07:06:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2306.17842/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this work, we introduce Semantic Pyramid AutoEncoder (SPAE) for enabling frozen LLMs to perform both understanding and generation tasks involving non-linguistic modalities such as images or videos. SPAE converts between raw pixels and interpretable lexical tokens (or words) extracted from the LLM's vocabulary. The resulting tokens capture both the semantic meaning and the fine-grained details needed for visual reconstruction, effectively translating the visual content into a language comprehensible to the LLM, and empowering it to perform a wide array of multimodal tasks. Our approach is va","authors_text":"Alexander G. Hauptmann, David A. Ross, Irfan Essa, Kevin Murphy, Lijun Yu, Lu Jiang, Ming-Hsuan Yang, Vivek Kumar, Wolfgang Macherey, Yanping Huang, Yonatan Bisk, Yong Cheng, Zhiruo Wang","cross_cats":["cs.CL","cs.MM"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-06-30T17:59:07Z","title":"SPAE: Semantic Pyramid AutoEncoder for Multimodal Generation with Frozen LLMs"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2306.17842","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b2e84b42d78cb54ea56eacf5639e5dbef8ef4bc075032dd26db8b8ef48d6ba94","target":"record","created_at":"2026-07-05T07:06:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a80664fbdc1bc2a64c5056df5ee581be6a2586cdd3468553d92c6acbcf0a2651","cross_cats_sorted":["cs.CL","cs.MM"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CV","submitted_at":"2023-06-30T17:59:07Z","title_canon_sha256":"ef98d20ca95e25fe3df86618ed7140094d082d1b1eff43a060917dabdae0448f"},"schema_version":"1.0","source":{"id":"2306.17842","kind":"arxiv","version":3}},"canonical_sha256":"cc8790f8c6ac56cc7c59aee494a909756a2c80286407cd11082ae945a0e35d34","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"cc8790f8c6ac56cc7c59aee494a909756a2c80286407cd11082ae945a0e35d34","first_computed_at":"2026-07-05T07:06:31.109303Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:06:31.109303Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"xMhYeZZh5X5GVOlH/BiFN+oBKfPCdhodRW0VW3H/z+hC3opmukrXO5wxi+MZyQvANzwnWmC4295hLO9wz5D7Dw==","signature_status":"signed_v1","signed_at":"2026-07-05T07:06:31.109818Z","signed_message":"canonical_sha256_bytes"},"source_id":"2306.17842","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b2e84b42d78cb54ea56eacf5639e5dbef8ef4bc075032dd26db8b8ef48d6ba94","sha256:69ce12c89007bbaeff0ee691363c3bd9d46faa26ce144e658365a5065b35817a"],"state_sha256":"304be345c5489d3386db53e4d1f7c68db1175b138d0ec62fee20b3338bd7ac29"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"863Q66Ry3lU9Z4bmp6CTAuC0UP9msqmwKp9u8PwRzGSHI05F2f9/lyGmRpmFebgcFRwi1Nt6L+MoR4scn31gAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T13:06:22.343447Z","bundle_sha256":"f191715cea73ed404f8ee357773504046d57d80ac8ddd946bc8f1031cff4deff"}}