{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:Z75XDQW6JGUUUH5KXE5A3UES7S","short_pith_number":"pith:Z75XDQW6","canonical_record":{"source":{"id":"2511.04355","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-11-06T13:38:03Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"14e9d3dea74032097fccbd832d17efd404ebd73d270b148ea01ca5a38ead5486","abstract_canon_sha256":"986453d1a989148997dd7f414a9cb9f67b9ddb5313af8513b7b994abb0a234b9"},"schema_version":"1.0"},"canonical_sha256":"cffb71c2de49a94a1faab93a0dd092fc945c99c02b44655aba8e9920c93e191b","source":{"kind":"arxiv","id":"2511.04355","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2511.04355","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"arxiv_version","alias_value":"2511.04355v1","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2511.04355","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"pith_short_12","alias_value":"Z75XDQW6JGUU","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"pith_short_16","alias_value":"Z75XDQW6JGUUUH5K","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"pith_short_8","alias_value":"Z75XDQW6","created_at":"2026-07-07T02:17:12Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:Z75XDQW6JGUUUH5KXE5A3UES7S","target":"record","payload":{"canonical_record":{"source":{"id":"2511.04355","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-11-06T13:38:03Z","cross_cats_sorted":["cs.LG"],"title_canon_sha256":"14e9d3dea74032097fccbd832d17efd404ebd73d270b148ea01ca5a38ead5486","abstract_canon_sha256":"986453d1a989148997dd7f414a9cb9f67b9ddb5313af8513b7b994abb0a234b9"},"schema_version":"1.0"},"canonical_sha256":"cffb71c2de49a94a1faab93a0dd092fc945c99c02b44655aba8e9920c93e191b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-07T02:17:12.973723Z","signature_b64":"ry0cJGxygTyFVCysEZrqHdP056IRRbCGjxPUGLiKcYFzgtvlY3e4T/Yo2FNW1stWhvSW+Tae8f4nb1lSxxNyDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"cffb71c2de49a94a1faab93a0dd092fc945c99c02b44655aba8e9920c93e191b","last_reissued_at":"2026-07-07T02:17:12.972688Z","signature_status":"signed_v1","first_computed_at":"2026-07-07T02:17:12.972688Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2511.04355","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-07T02:17:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dR1u/VMr+7RAoSua2Dr36Chv/M8t38Td587qyPJowXAfMvLPUQgm9zVLAmaNF5J6P5YgDB3sL0O7Bsa4H0RVBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T17:14:20.344161Z"},"content_sha256":"503794720947207042d7508ed2a8848c41e49277b5b79fc313749d65b27dc743","schema_version":"1.0","event_id":"sha256:503794720947207042d7508ed2a8848c41e49277b5b79fc313749d65b27dc743"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:Z75XDQW6JGUUUH5KXE5A3UES7S","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG"],"primary_cat":"cs.SE","authors_text":"Amir Molzam Sharifloo, Daniel Maninger, Maedeh Heydari, Mira Mezini, Parsa Kazerooni","submitted_at":"2025-11-06T13:38:03Z","abstract_excerpt":"Large Language Models (LLMs) have achieved remarkable success in code generation, and the race to improve their performance has become a central focus of AI research. Benchmarks and leaderboards are increasingly popular, offering quantitative rankings of LLMs. However, they provide limited insight into the tasks that LLMs consistently fail to solve - information that is crucial for understanding current limitations and guiding the development of more capable models. To address this gap, we examined code generation tasks across four popular benchmarks, identifying those that major LLMs are most"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2511.04355","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2511.04355/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-07T02:17:12Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Eet0UuEb+bAGDjaUDCe4irxbIQwuxyQM7pUX0G+/JlY5wcyjG1NKSUtwAzQGDsqudD+VMjtQchBACztpeEnwAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T17:14:20.344761Z"},"content_sha256":"fe6df5cdf537258495684985a631d8c88d9e2ea2a921388dc1d3340e0f233b7e","schema_version":"1.0","event_id":"sha256:fe6df5cdf537258495684985a631d8c88d9e2ea2a921388dc1d3340e0f233b7e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Z75XDQW6JGUUUH5KXE5A3UES7S/bundle.json","state_url":"https://pith.science/pith/Z75XDQW6JGUUUH5KXE5A3UES7S/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Z75XDQW6JGUUUH5KXE5A3UES7S/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T17:14:20Z","links":{"resolver":"https://pith.science/pith/Z75XDQW6JGUUUH5KXE5A3UES7S","bundle":"https://pith.science/pith/Z75XDQW6JGUUUH5KXE5A3UES7S/bundle.json","state":"https://pith.science/pith/Z75XDQW6JGUUUH5KXE5A3UES7S/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Z75XDQW6JGUUUH5KXE5A3UES7S/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:Z75XDQW6JGUUUH5KXE5A3UES7S","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"986453d1a989148997dd7f414a9cb9f67b9ddb5313af8513b7b994abb0a234b9","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-11-06T13:38:03Z","title_canon_sha256":"14e9d3dea74032097fccbd832d17efd404ebd73d270b148ea01ca5a38ead5486"},"schema_version":"1.0","source":{"id":"2511.04355","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2511.04355","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"arxiv_version","alias_value":"2511.04355v1","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2511.04355","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"pith_short_12","alias_value":"Z75XDQW6JGUU","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"pith_short_16","alias_value":"Z75XDQW6JGUUUH5K","created_at":"2026-07-07T02:17:12Z"},{"alias_kind":"pith_short_8","alias_value":"Z75XDQW6","created_at":"2026-07-07T02:17:12Z"}],"graph_snapshots":[{"event_id":"sha256:fe6df5cdf537258495684985a631d8c88d9e2ea2a921388dc1d3340e0f233b7e","target":"graph","created_at":"2026-07-07T02:17:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2511.04355/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have achieved remarkable success in code generation, and the race to improve their performance has become a central focus of AI research. Benchmarks and leaderboards are increasingly popular, offering quantitative rankings of LLMs. However, they provide limited insight into the tasks that LLMs consistently fail to solve - information that is crucial for understanding current limitations and guiding the development of more capable models. To address this gap, we examined code generation tasks across four popular benchmarks, identifying those that major LLMs are most","authors_text":"Amir Molzam Sharifloo, Daniel Maninger, Maedeh Heydari, Mira Mezini, Parsa Kazerooni","cross_cats":["cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-11-06T13:38:03Z","title":"Where Do LLMs Still Struggle? An In-Depth Analysis of Code Generation Benchmarks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2511.04355","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:503794720947207042d7508ed2a8848c41e49277b5b79fc313749d65b27dc743","target":"record","created_at":"2026-07-07T02:17:12Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"986453d1a989148997dd7f414a9cb9f67b9ddb5313af8513b7b994abb0a234b9","cross_cats_sorted":["cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-11-06T13:38:03Z","title_canon_sha256":"14e9d3dea74032097fccbd832d17efd404ebd73d270b148ea01ca5a38ead5486"},"schema_version":"1.0","source":{"id":"2511.04355","kind":"arxiv","version":1}},"canonical_sha256":"cffb71c2de49a94a1faab93a0dd092fc945c99c02b44655aba8e9920c93e191b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"cffb71c2de49a94a1faab93a0dd092fc945c99c02b44655aba8e9920c93e191b","first_computed_at":"2026-07-07T02:17:12.972688Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-07T02:17:12.972688Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ry0cJGxygTyFVCysEZrqHdP056IRRbCGjxPUGLiKcYFzgtvlY3e4T/Yo2FNW1stWhvSW+Tae8f4nb1lSxxNyDg==","signature_status":"signed_v1","signed_at":"2026-07-07T02:17:12.973723Z","signed_message":"canonical_sha256_bytes"},"source_id":"2511.04355","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:503794720947207042d7508ed2a8848c41e49277b5b79fc313749d65b27dc743","sha256:fe6df5cdf537258495684985a631d8c88d9e2ea2a921388dc1d3340e0f233b7e"],"state_sha256":"9875714883cf115c50e0824ccc6b4e97f9eee0d5005d1a11ff94d069e5fddd6b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"f763wI/IWQQQZuweF3lrbbbI9Vbs/m4Z+TP1yUvOTaL+XO4rIHuTjaYp69HYcTTOa0HDCv0TFYPmLrZRwlGxDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T17:14:20.349608Z","bundle_sha256":"6e4889cffbd64c478bd1efab60156b87586ea72869458ed2f8fe0fa0ebe295d2"}}