{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:CCWHYHTL72LUIORYFKHYMO7J2M","short_pith_number":"pith:CCWHYHTL","canonical_record":{"source":{"id":"2410.01769","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-02T17:25:37Z","cross_cats_sorted":[],"title_canon_sha256":"73dcde8354fc61635d7bb587495e79be02f2d38d45de07756440a0214dd88b1a","abstract_canon_sha256":"55ccf81e8356f8beacf54f77ed31ec32d4cd5431b5f0d0974bfbbd3d14afe6f9"},"schema_version":"1.0"},"canonical_sha256":"10ac7c1e6bfe97443a382a8f863be9d323426d3966b9975eb22b042c11b366c7","source":{"kind":"arxiv","id":"2410.01769","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.01769","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"arxiv_version","alias_value":"2410.01769v2","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.01769","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_12","alias_value":"CCWHYHTL72LU","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_16","alias_value":"CCWHYHTL72LUIORY","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_8","alias_value":"CCWHYHTL","created_at":"2026-07-05T09:15:16Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:CCWHYHTL72LUIORYFKHYMO7J2M","target":"record","payload":{"canonical_record":{"source":{"id":"2410.01769","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-02T17:25:37Z","cross_cats_sorted":[],"title_canon_sha256":"73dcde8354fc61635d7bb587495e79be02f2d38d45de07756440a0214dd88b1a","abstract_canon_sha256":"55ccf81e8356f8beacf54f77ed31ec32d4cd5431b5f0d0974bfbbd3d14afe6f9"},"schema_version":"1.0"},"canonical_sha256":"10ac7c1e6bfe97443a382a8f863be9d323426d3966b9975eb22b042c11b366c7","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:15:16.622804Z","signature_b64":"L5J2UugrodpwHdlBai9inGydkcRhrTQmNkdEKyQrsw9OeIDWOoMKaIeX3wB5aSkkRk4bW/pItJkzbdGEI3vWDA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"10ac7c1e6bfe97443a382a8f863be9d323426d3966b9975eb22b042c11b366c7","last_reissued_at":"2026-07-05T09:15:16.622302Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:15:16.622302Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.01769","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:15:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"S/tmJe5LHD0pi3PqPnwTPtwqXsVCdiZ0n4EC6/P09KJSTQei21ChZJt+OWuYKxMpLiRyrTCL/hJ1Y1DpHn1uDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T15:50:51.050535Z"},"content_sha256":"c091a205f9c0cf66ed8b0804f2d6bfb352ecd8b4bdd2b6b6d36da0377da59076","schema_version":"1.0","event_id":"sha256:c091a205f9c0cf66ed8b0804f2d6bfb352ecd8b4bdd2b6b6d36da0377da59076"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:CCWHYHTL72LUIORYFKHYMO7J2M","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Quantifying Generalization Complexity for Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Himabindu Lakkaraju, Hongyin Luo, James Glass, Xiangjun Fan, Xuliang Huang, Yibo Jiang, Zhenting Qi, Zhuokai Zhao","submitted_at":"2024-10-02T17:25:37Z","abstract_excerpt":"While large language models (LLMs) have shown exceptional capabilities in understanding complex queries and performing sophisticated tasks, their generalization abilities are often deeply entangled with memorization, necessitating more precise evaluation. To address this challenge, we introduce Scylla, a dynamic evaluation framework that quantitatively measures the generalization abilities of LLMs. Scylla disentangles generalization from memorization via assessing model performance on both in-distribution (ID) and out-of-distribution (OOD) data through 20 tasks across 5 levels of complexity. T"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.01769","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.01769/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:15:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"knzr7+/sxi/x/dzbROw4poqpKSpZx5NGXXCg2qpweJvJ6vo6w4vGElBDL5Xh/Pzec93w6fBNGzF5b+NAvzImBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T15:50:51.051037Z"},"content_sha256":"241b3c9da485153f69ee6575bcae1486885d94ce6c62ceff20cdc1f3cb03be30","schema_version":"1.0","event_id":"sha256:241b3c9da485153f69ee6575bcae1486885d94ce6c62ceff20cdc1f3cb03be30"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/CCWHYHTL72LUIORYFKHYMO7J2M/bundle.json","state_url":"https://pith.science/pith/CCWHYHTL72LUIORYFKHYMO7J2M/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/CCWHYHTL72LUIORYFKHYMO7J2M/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T15:50:51Z","links":{"resolver":"https://pith.science/pith/CCWHYHTL72LUIORYFKHYMO7J2M","bundle":"https://pith.science/pith/CCWHYHTL72LUIORYFKHYMO7J2M/bundle.json","state":"https://pith.science/pith/CCWHYHTL72LUIORYFKHYMO7J2M/state.json","well_known_bundle":"https://pith.science/.well-known/pith/CCWHYHTL72LUIORYFKHYMO7J2M/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:CCWHYHTL72LUIORYFKHYMO7J2M","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"55ccf81e8356f8beacf54f77ed31ec32d4cd5431b5f0d0974bfbbd3d14afe6f9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-02T17:25:37Z","title_canon_sha256":"73dcde8354fc61635d7bb587495e79be02f2d38d45de07756440a0214dd88b1a"},"schema_version":"1.0","source":{"id":"2410.01769","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.01769","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"arxiv_version","alias_value":"2410.01769v2","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.01769","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_12","alias_value":"CCWHYHTL72LU","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_16","alias_value":"CCWHYHTL72LUIORY","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_8","alias_value":"CCWHYHTL","created_at":"2026-07-05T09:15:16Z"}],"graph_snapshots":[{"event_id":"sha256:241b3c9da485153f69ee6575bcae1486885d94ce6c62ceff20cdc1f3cb03be30","target":"graph","created_at":"2026-07-05T09:15:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.01769/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While large language models (LLMs) have shown exceptional capabilities in understanding complex queries and performing sophisticated tasks, their generalization abilities are often deeply entangled with memorization, necessitating more precise evaluation. To address this challenge, we introduce Scylla, a dynamic evaluation framework that quantitatively measures the generalization abilities of LLMs. Scylla disentangles generalization from memorization via assessing model performance on both in-distribution (ID) and out-of-distribution (OOD) data through 20 tasks across 5 levels of complexity. T","authors_text":"Himabindu Lakkaraju, Hongyin Luo, James Glass, Xiangjun Fan, Xuliang Huang, Yibo Jiang, Zhenting Qi, Zhuokai Zhao","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-02T17:25:37Z","title":"Quantifying Generalization Complexity for Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.01769","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c091a205f9c0cf66ed8b0804f2d6bfb352ecd8b4bdd2b6b6d36da0377da59076","target":"record","created_at":"2026-07-05T09:15:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"55ccf81e8356f8beacf54f77ed31ec32d4cd5431b5f0d0974bfbbd3d14afe6f9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-02T17:25:37Z","title_canon_sha256":"73dcde8354fc61635d7bb587495e79be02f2d38d45de07756440a0214dd88b1a"},"schema_version":"1.0","source":{"id":"2410.01769","kind":"arxiv","version":2}},"canonical_sha256":"10ac7c1e6bfe97443a382a8f863be9d323426d3966b9975eb22b042c11b366c7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"10ac7c1e6bfe97443a382a8f863be9d323426d3966b9975eb22b042c11b366c7","first_computed_at":"2026-07-05T09:15:16.622302Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:15:16.622302Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"L5J2UugrodpwHdlBai9inGydkcRhrTQmNkdEKyQrsw9OeIDWOoMKaIeX3wB5aSkkRk4bW/pItJkzbdGEI3vWDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:15:16.622804Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.01769","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c091a205f9c0cf66ed8b0804f2d6bfb352ecd8b4bdd2b6b6d36da0377da59076","sha256:241b3c9da485153f69ee6575bcae1486885d94ce6c62ceff20cdc1f3cb03be30"],"state_sha256":"9ccec31be202dbc90df76398f1ab67040caba615284ee1b8e8c2d21a88e3e179"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"L9LPGo8pL7zwVXXLe9HVAC7XAuNqOOCLCQat870fQ73HSf1IXCryZ5eoLlQkgdgBs9QnaiMn8eJZ7XkKdJ8ZCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T15:50:51.055007Z","bundle_sha256":"4854bb4a6f1d89a120efc7db219e657f65141fd8635b64ab25d43b3bbfae5790"}}