{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:YDBPSDPIJDB2XQYQ7G7GX2YIZJ","short_pith_number":"pith:YDBPSDPI","canonical_record":{"source":{"id":"2408.03160","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CV","submitted_at":"2024-08-04T06:12:42Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"bb6ca1152c3e4a7c0375de34af1ed51de962541b4fb3f86fb325ab981303596d","abstract_canon_sha256":"7eca5554f972a1a463d4a6f084febfb38d45be8c375993a377579e8696f9b3cd"},"schema_version":"1.0"},"canonical_sha256":"c0c2f90de848c3abc310f9be6beb08ca7c9ace7eedce8d24e099dbb961fc4692","source":{"kind":"arxiv","id":"2408.03160","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.03160","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"arxiv_version","alias_value":"2408.03160v2","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.03160","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"pith_short_12","alias_value":"YDBPSDPIJDB2","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"pith_short_16","alias_value":"YDBPSDPIJDB2XQYQ","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"pith_short_8","alias_value":"YDBPSDPI","created_at":"2026-07-05T08:54:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:YDBPSDPIJDB2XQYQ7G7GX2YIZJ","target":"record","payload":{"canonical_record":{"source":{"id":"2408.03160","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CV","submitted_at":"2024-08-04T06:12:42Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"bb6ca1152c3e4a7c0375de34af1ed51de962541b4fb3f86fb325ab981303596d","abstract_canon_sha256":"7eca5554f972a1a463d4a6f084febfb38d45be8c375993a377579e8696f9b3cd"},"schema_version":"1.0"},"canonical_sha256":"c0c2f90de848c3abc310f9be6beb08ca7c9ace7eedce8d24e099dbb961fc4692","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:54:55.075254Z","signature_b64":"D/E+Hxc8nxdSlbBrlCZYejbMlq5ABBYPVLp5SavEzcUwhB771WRDoH2QGK8TIT3+HJx8viM1k0EieJ1nxUnlAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c0c2f90de848c3abc310f9be6beb08ca7c9ace7eedce8d24e099dbb961fc4692","last_reissued_at":"2026-07-05T08:54:55.074809Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:54:55.074809Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.03160","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:54:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"z6lMI7ax7ZpiJo2sV8SFs9FmTJyG8QCpLk0BVa+NunzH1DdASZbsnzn5AiD+/9wonAWjv11CS6ilOmxxz+8YAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T08:15:57.414446Z"},"content_sha256":"84366efcd598678d31a556f9c9dadcb819af2fa3b4e59b91f8521a9b5f851aa8","schema_version":"1.0","event_id":"sha256:84366efcd598678d31a556f9c9dadcb819af2fa3b4e59b91f8521a9b5f851aa8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:YDBPSDPIJDB2XQYQ7G7GX2YIZJ","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"User-in-the-loop Evaluation of Multimodal LLMs for Activity Assistance","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CV","authors_text":"Brian Chen, Hamid Eghbalzadeh, Mrinal Verghese, Ruta Desai, Tushar Nagarajan","submitted_at":"2024-08-04T06:12:42Z","abstract_excerpt":"Our research investigates the capability of modern multimodal reasoning models, powered by Large Language Models (LLMs), to facilitate vision-powered assistants for multi-step daily activities. Such assistants must be able to 1) encode relevant visual history from the assistant's sensors, e.g., camera, 2) forecast future actions for accomplishing the activity, and 3) replan based on the user in the loop. To evaluate the first two capabilities, grounding visual history and forecasting in short and long horizons, we conduct benchmarking of two prominent classes of multimodal LLM approaches -- So"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.03160","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.03160/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:54:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YdXeIWGGoyVB283d8ybSMQqFNtqL9K0l+kDBPqpaQwieWDsLDtiBHR2Phj71BZOzXWRXFPIyu6raqoVH+IzKAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-01T08:15:57.415326Z"},"content_sha256":"e9b378264d3ef886ef91e16e285b4a0656238b3a892c64cb3e4ed7d980b93d18","schema_version":"1.0","event_id":"sha256:e9b378264d3ef886ef91e16e285b4a0656238b3a892c64cb3e4ed7d980b93d18"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YDBPSDPIJDB2XQYQ7G7GX2YIZJ/bundle.json","state_url":"https://pith.science/pith/YDBPSDPIJDB2XQYQ7G7GX2YIZJ/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YDBPSDPIJDB2XQYQ7G7GX2YIZJ/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-01T08:15:57Z","links":{"resolver":"https://pith.science/pith/YDBPSDPIJDB2XQYQ7G7GX2YIZJ","bundle":"https://pith.science/pith/YDBPSDPIJDB2XQYQ7G7GX2YIZJ/bundle.json","state":"https://pith.science/pith/YDBPSDPIJDB2XQYQ7G7GX2YIZJ/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YDBPSDPIJDB2XQYQ7G7GX2YIZJ/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:YDBPSDPIJDB2XQYQ7G7GX2YIZJ","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7eca5554f972a1a463d4a6f084febfb38d45be8c375993a377579e8696f9b3cd","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CV","submitted_at":"2024-08-04T06:12:42Z","title_canon_sha256":"bb6ca1152c3e4a7c0375de34af1ed51de962541b4fb3f86fb325ab981303596d"},"schema_version":"1.0","source":{"id":"2408.03160","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.03160","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"arxiv_version","alias_value":"2408.03160v2","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.03160","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"pith_short_12","alias_value":"YDBPSDPIJDB2","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"pith_short_16","alias_value":"YDBPSDPIJDB2XQYQ","created_at":"2026-07-05T08:54:55Z"},{"alias_kind":"pith_short_8","alias_value":"YDBPSDPI","created_at":"2026-07-05T08:54:55Z"}],"graph_snapshots":[{"event_id":"sha256:e9b378264d3ef886ef91e16e285b4a0656238b3a892c64cb3e4ed7d980b93d18","target":"graph","created_at":"2026-07-05T08:54:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.03160/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Our research investigates the capability of modern multimodal reasoning models, powered by Large Language Models (LLMs), to facilitate vision-powered assistants for multi-step daily activities. Such assistants must be able to 1) encode relevant visual history from the assistant's sensors, e.g., camera, 2) forecast future actions for accomplishing the activity, and 3) replan based on the user in the loop. To evaluate the first two capabilities, grounding visual history and forecasting in short and long horizons, we conduct benchmarking of two prominent classes of multimodal LLM approaches -- So","authors_text":"Brian Chen, Hamid Eghbalzadeh, Mrinal Verghese, Ruta Desai, Tushar Nagarajan","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CV","submitted_at":"2024-08-04T06:12:42Z","title":"User-in-the-loop Evaluation of Multimodal LLMs for Activity Assistance"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.03160","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:84366efcd598678d31a556f9c9dadcb819af2fa3b4e59b91f8521a9b5f851aa8","target":"record","created_at":"2026-07-05T08:54:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7eca5554f972a1a463d4a6f084febfb38d45be8c375993a377579e8696f9b3cd","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.CV","submitted_at":"2024-08-04T06:12:42Z","title_canon_sha256":"bb6ca1152c3e4a7c0375de34af1ed51de962541b4fb3f86fb325ab981303596d"},"schema_version":"1.0","source":{"id":"2408.03160","kind":"arxiv","version":2}},"canonical_sha256":"c0c2f90de848c3abc310f9be6beb08ca7c9ace7eedce8d24e099dbb961fc4692","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c0c2f90de848c3abc310f9be6beb08ca7c9ace7eedce8d24e099dbb961fc4692","first_computed_at":"2026-07-05T08:54:55.074809Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:54:55.074809Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"D/E+Hxc8nxdSlbBrlCZYejbMlq5ABBYPVLp5SavEzcUwhB771WRDoH2QGK8TIT3+HJx8viM1k0EieJ1nxUnlAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T08:54:55.075254Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.03160","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:84366efcd598678d31a556f9c9dadcb819af2fa3b4e59b91f8521a9b5f851aa8","sha256:e9b378264d3ef886ef91e16e285b4a0656238b3a892c64cb3e4ed7d980b93d18"],"state_sha256":"35137cede71b0fdb0ef178739f2d002f0b96ac5747d5d562e6e6ad108cb26943"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"cuEWKDY9Xq+T1nYs5pY0QvLRBX5Bww9g5K7K8g4QLAi7/53hGO96rqUiPlqOXEffoydZzy1UP+8i8Et1JqVhDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-01T08:15:57.419990Z","bundle_sha256":"5aedd904719f3d18208eac1830833a9e2779a6cc9b30b084b9d2c9c64d59ca4e"}}