{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:CCWHYHTL72LUIORYFKHYMO7J2M","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"55ccf81e8356f8beacf54f77ed31ec32d4cd5431b5f0d0974bfbbd3d14afe6f9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-02T17:25:37Z","title_canon_sha256":"73dcde8354fc61635d7bb587495e79be02f2d38d45de07756440a0214dd88b1a"},"schema_version":"1.0","source":{"id":"2410.01769","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.01769","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"arxiv_version","alias_value":"2410.01769v2","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.01769","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_12","alias_value":"CCWHYHTL72LU","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_16","alias_value":"CCWHYHTL72LUIORY","created_at":"2026-07-05T09:15:16Z"},{"alias_kind":"pith_short_8","alias_value":"CCWHYHTL","created_at":"2026-07-05T09:15:16Z"}],"graph_snapshots":[{"event_id":"sha256:241b3c9da485153f69ee6575bcae1486885d94ce6c62ceff20cdc1f3cb03be30","target":"graph","created_at":"2026-07-05T09:15:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.01769/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While large language models (LLMs) have shown exceptional capabilities in understanding complex queries and performing sophisticated tasks, their generalization abilities are often deeply entangled with memorization, necessitating more precise evaluation. To address this challenge, we introduce Scylla, a dynamic evaluation framework that quantitatively measures the generalization abilities of LLMs. Scylla disentangles generalization from memorization via assessing model performance on both in-distribution (ID) and out-of-distribution (OOD) data through 20 tasks across 5 levels of complexity. T","authors_text":"Himabindu Lakkaraju, Hongyin Luo, James Glass, Xiangjun Fan, Xuliang Huang, Yibo Jiang, Zhenting Qi, Zhuokai Zhao","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-02T17:25:37Z","title":"Quantifying Generalization Complexity for Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.01769","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:c091a205f9c0cf66ed8b0804f2d6bfb352ecd8b4bdd2b6b6d36da0377da59076","target":"record","created_at":"2026-07-05T09:15:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"55ccf81e8356f8beacf54f77ed31ec32d4cd5431b5f0d0974bfbbd3d14afe6f9","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-10-02T17:25:37Z","title_canon_sha256":"73dcde8354fc61635d7bb587495e79be02f2d38d45de07756440a0214dd88b1a"},"schema_version":"1.0","source":{"id":"2410.01769","kind":"arxiv","version":2}},"canonical_sha256":"10ac7c1e6bfe97443a382a8f863be9d323426d3966b9975eb22b042c11b366c7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"10ac7c1e6bfe97443a382a8f863be9d323426d3966b9975eb22b042c11b366c7","first_computed_at":"2026-07-05T09:15:16.622302Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:15:16.622302Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"L5J2UugrodpwHdlBai9inGydkcRhrTQmNkdEKyQrsw9OeIDWOoMKaIeX3wB5aSkkRk4bW/pItJkzbdGEI3vWDA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:15:16.622804Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.01769","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:c091a205f9c0cf66ed8b0804f2d6bfb352ecd8b4bdd2b6b6d36da0377da59076","sha256:241b3c9da485153f69ee6575bcae1486885d94ce6c62ceff20cdc1f3cb03be30"],"state_sha256":"9ccec31be202dbc90df76398f1ab67040caba615284ee1b8e8c2d21a88e3e179"}