{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:HJJNK7BUDS6KORYBDWKEJ3I2GH","short_pith_number":"pith:HJJNK7BU","canonical_record":{"source":{"id":"2504.05500","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-07T20:53:18Z","cross_cats_sorted":["cs.LG","cs.SE"],"title_canon_sha256":"4a933a177102affe499bc281c884e972744ca6ffab4968cf91c8801633a90f66","abstract_canon_sha256":"081bd12c8042ea5491f999045c48b4d1a0c7289d78c382acc766d1532855d271"},"schema_version":"1.0"},"canonical_sha256":"3a52d57c341cbca747011d9444ed1a31e20cfa0190e8dd36d34277c25ec570f1","source":{"kind":"arxiv","id":"2504.05500","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.05500","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"arxiv_version","alias_value":"2504.05500v2","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.05500","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"pith_short_12","alias_value":"HJJNK7BUDS6K","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"pith_short_16","alias_value":"HJJNK7BUDS6KORYB","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"pith_short_8","alias_value":"HJJNK7BU","created_at":"2026-07-05T10:47:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:HJJNK7BUDS6KORYBDWKEJ3I2GH","target":"record","payload":{"canonical_record":{"source":{"id":"2504.05500","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-07T20:53:18Z","cross_cats_sorted":["cs.LG","cs.SE"],"title_canon_sha256":"4a933a177102affe499bc281c884e972744ca6ffab4968cf91c8801633a90f66","abstract_canon_sha256":"081bd12c8042ea5491f999045c48b4d1a0c7289d78c382acc766d1532855d271"},"schema_version":"1.0"},"canonical_sha256":"3a52d57c341cbca747011d9444ed1a31e20cfa0190e8dd36d34277c25ec570f1","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:47:02.412646Z","signature_b64":"4aZ02a+JwybWEfGF87q2l7uA7sc8qKbdE0Y3ab+Tuc3QZudh0/YEXBFpf5c3qVebV8nrTcBytMtet8nup4n0Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3a52d57c341cbca747011d9444ed1a31e20cfa0190e8dd36d34277c25ec570f1","last_reissued_at":"2026-07-05T10:47:02.412066Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:47:02.412066Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2504.05500","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:47:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"H2qHh8GVRiN9qibEGxMj3zoKYeZS2Nfb8yi9TmKrrSC9IlPCA9VLUAF/typ13J/87bOVfvLjwYpXeMn6ExQiDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T10:09:15.478843Z"},"content_sha256":"81d53374ea3f9d7358640e24a22498a6a0b79b56b60ce13e0b527443af6e6b5a","schema_version":"1.0","event_id":"sha256:81d53374ea3f9d7358640e24a22498a6a0b79b56b60ce13e0b527443af6e6b5a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:HJJNK7BUDS6KORYBDWKEJ3I2GH","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Prism: Dynamic and Flexible Benchmarking of LLMs Code Generation with Monte Carlo Tree Search","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.LG","cs.SE"],"primary_cat":"cs.AI","authors_text":"Amin Nikanjam, Foutse Khomh, Vahid Majdinasab","submitted_at":"2025-04-07T20:53:18Z","abstract_excerpt":"The rapid advancement of Large Language Models (LLMs) has outpaced traditional evaluation methods. Static benchmarks fail to capture the depth and breadth of LLM capabilities and eventually become obsolete, while most dynamic approaches either rely too heavily on LLM-based evaluation or remain constrained by predefined test sets. We introduce Prism, a flexible, dynamic benchmarking framework designed for comprehensive LLM assessment. Prism builds on three key components: (1) a tree-based state representation that models evaluation as a Markov Decision Process, (2) a Monte Carlo Tree Search alg"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.05500","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.05500/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:47:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4bkx0RaMzEAqVUv89MkOdpo4pUqFPTjrjhfxXbIwShKBQiYx5Swbfloae6WLaCEnr2XiCQJRIMFnr+UKaAzQCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T10:09:15.479229Z"},"content_sha256":"70d0b6336950a0d78fe394f0c9426c3925af79138f879b65cbbdb0e7339ab165","schema_version":"1.0","event_id":"sha256:70d0b6336950a0d78fe394f0c9426c3925af79138f879b65cbbdb0e7339ab165"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/HJJNK7BUDS6KORYBDWKEJ3I2GH/bundle.json","state_url":"https://pith.science/pith/HJJNK7BUDS6KORYBDWKEJ3I2GH/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/HJJNK7BUDS6KORYBDWKEJ3I2GH/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T10:09:15Z","links":{"resolver":"https://pith.science/pith/HJJNK7BUDS6KORYBDWKEJ3I2GH","bundle":"https://pith.science/pith/HJJNK7BUDS6KORYBDWKEJ3I2GH/bundle.json","state":"https://pith.science/pith/HJJNK7BUDS6KORYBDWKEJ3I2GH/state.json","well_known_bundle":"https://pith.science/.well-known/pith/HJJNK7BUDS6KORYBDWKEJ3I2GH/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:HJJNK7BUDS6KORYBDWKEJ3I2GH","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"081bd12c8042ea5491f999045c48b4d1a0c7289d78c382acc766d1532855d271","cross_cats_sorted":["cs.LG","cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-07T20:53:18Z","title_canon_sha256":"4a933a177102affe499bc281c884e972744ca6ffab4968cf91c8801633a90f66"},"schema_version":"1.0","source":{"id":"2504.05500","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.05500","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"arxiv_version","alias_value":"2504.05500v2","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.05500","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"pith_short_12","alias_value":"HJJNK7BUDS6K","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"pith_short_16","alias_value":"HJJNK7BUDS6KORYB","created_at":"2026-07-05T10:47:02Z"},{"alias_kind":"pith_short_8","alias_value":"HJJNK7BU","created_at":"2026-07-05T10:47:02Z"}],"graph_snapshots":[{"event_id":"sha256:70d0b6336950a0d78fe394f0c9426c3925af79138f879b65cbbdb0e7339ab165","target":"graph","created_at":"2026-07-05T10:47:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2504.05500/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The rapid advancement of Large Language Models (LLMs) has outpaced traditional evaluation methods. Static benchmarks fail to capture the depth and breadth of LLM capabilities and eventually become obsolete, while most dynamic approaches either rely too heavily on LLM-based evaluation or remain constrained by predefined test sets. We introduce Prism, a flexible, dynamic benchmarking framework designed for comprehensive LLM assessment. Prism builds on three key components: (1) a tree-based state representation that models evaluation as a Markov Decision Process, (2) a Monte Carlo Tree Search alg","authors_text":"Amin Nikanjam, Foutse Khomh, Vahid Majdinasab","cross_cats":["cs.LG","cs.SE"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-07T20:53:18Z","title":"Prism: Dynamic and Flexible Benchmarking of LLMs Code Generation with Monte Carlo Tree Search"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.05500","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:81d53374ea3f9d7358640e24a22498a6a0b79b56b60ce13e0b527443af6e6b5a","target":"record","created_at":"2026-07-05T10:47:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"081bd12c8042ea5491f999045c48b4d1a0c7289d78c382acc766d1532855d271","cross_cats_sorted":["cs.LG","cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2025-04-07T20:53:18Z","title_canon_sha256":"4a933a177102affe499bc281c884e972744ca6ffab4968cf91c8801633a90f66"},"schema_version":"1.0","source":{"id":"2504.05500","kind":"arxiv","version":2}},"canonical_sha256":"3a52d57c341cbca747011d9444ed1a31e20cfa0190e8dd36d34277c25ec570f1","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3a52d57c341cbca747011d9444ed1a31e20cfa0190e8dd36d34277c25ec570f1","first_computed_at":"2026-07-05T10:47:02.412066Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:47:02.412066Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"4aZ02a+JwybWEfGF87q2l7uA7sc8qKbdE0Y3ab+Tuc3QZudh0/YEXBFpf5c3qVebV8nrTcBytMtet8nup4n0Bg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:47:02.412646Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.05500","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:81d53374ea3f9d7358640e24a22498a6a0b79b56b60ce13e0b527443af6e6b5a","sha256:70d0b6336950a0d78fe394f0c9426c3925af79138f879b65cbbdb0e7339ab165"],"state_sha256":"0ae3e2925990042d6040215accbb1c7791d84529a5a09a802de76e1b344e69ea"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YLB9vB40Fy2mHnXpBoDx/uZbwuCPneTvEPurB+brXYNpKreLkUi/Ev5WJ3WLkcW2xyH+/S+8LLgXZaSpaBIjBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T10:09:15.481146Z","bundle_sha256":"2e65a5e63dd5d4c4975a225195b2d75915c5bee9dd31b06873fdf553faa3e0ae"}}