{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2026:Q673SXFMAXHU5HRT6EKSAGSILK","short_pith_number":"pith:Q673SXFM","canonical_record":{"source":{"id":"2607.13820","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-07-15T13:28:18Z","cross_cats_sorted":[],"title_canon_sha256":"03ea9ecea1b2ed4cfcf95d0b84f75d248853b6f23f4e39d54189b250e23868ba","abstract_canon_sha256":"7f538b77cc93d783609f35dd6f255e5dad1ca91e1e4506e56d1312b30b6fd0b4"},"schema_version":"1.0"},"canonical_sha256":"87bfb95cac05cf4e9e33f115201a485a8d1b03c4b41ace4d9577bcb8900520a5","source":{"kind":"arxiv","id":"2607.13820","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2607.13820","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"arxiv_version","alias_value":"2607.13820v1","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2607.13820","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"pith_short_12","alias_value":"Q673SXFMAXHU","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"pith_short_16","alias_value":"Q673SXFMAXHU5HRT","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"pith_short_8","alias_value":"Q673SXFM","created_at":"2026-07-16T01:23:08Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2026:Q673SXFMAXHU5HRT6EKSAGSILK","target":"record","payload":{"canonical_record":{"source":{"id":"2607.13820","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-07-15T13:28:18Z","cross_cats_sorted":[],"title_canon_sha256":"03ea9ecea1b2ed4cfcf95d0b84f75d248853b6f23f4e39d54189b250e23868ba","abstract_canon_sha256":"7f538b77cc93d783609f35dd6f255e5dad1ca91e1e4506e56d1312b30b6fd0b4"},"schema_version":"1.0"},"canonical_sha256":"87bfb95cac05cf4e9e33f115201a485a8d1b03c4b41ace4d9577bcb8900520a5","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-16T01:23:08.095706Z","signature_b64":"TcgFu9IXlwNuTZ7hYgOqASHpCqidriFLVpyrJKaoZzvEePscfxfGj74Jl33o7cuoJutu9prvn472H0R0K2v7Bg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"87bfb95cac05cf4e9e33f115201a485a8d1b03c4b41ace4d9577bcb8900520a5","last_reissued_at":"2026-07-16T01:23:08.094900Z","signature_status":"signed_v1","first_computed_at":"2026-07-16T01:23:08.094900Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2607.13820","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-16T01:23:08Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KHjSCnhA5P4P1yDx9Z0dEy59JgRqIirzTeL53wYnhrv4eTRfb7pm+YW9eetSEaPIM5OSoXg8DEKOyxBFsHK/Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T00:25:49.310287Z"},"content_sha256":"8982bfe910200e0aa0b3d693016a57b0e1c22bb1eedf712fb7ab062bdf54c502","schema_version":"1.0","event_id":"sha256:8982bfe910200e0aa0b3d693016a57b0e1c22bb1eedf712fb7ab062bdf54c502"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2026:Q673SXFMAXHU5HRT6EKSAGSILK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"PROBE: Benchmarking Code Generation in Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.SE","authors_text":"Jo\\~ao R. Campos, Marco Vieira, Rodrigo Pato Nogueira","submitted_at":"2026-07-15T13:28:18Z","abstract_excerpt":"Large Language Models (LLMs) are increasingly being used in everyday software engineering tasks, particularly in automated code generation. Despite their widespread adoption, these models remain far from perfect, making systematic and fair evaluation essential to understand their strengths and limitations. In the context of code generation, existing benchmarks are limited: they often target a single programming language and rely primarily on unit test outcomes, while overlooking other critical dimensions such as the overall quality of the generated code and its closeness to a valid solution. T"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2607.13820","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2607.13820/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-16T01:23:08Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"/X3StOvxv1FXVz1f1byleR82urDJf1ZoD//NPGh62g50w+CCGf990RkbAwoapjdwnB9AV+xfmSN8VcarcZzkAQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T00:25:49.310667Z"},"content_sha256":"ca59d2cf684a32318166642b7aaea56fe025865325f694d12a9488bc2df3b256","schema_version":"1.0","event_id":"sha256:ca59d2cf684a32318166642b7aaea56fe025865325f694d12a9488bc2df3b256"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Q673SXFMAXHU5HRT6EKSAGSILK/bundle.json","state_url":"https://pith.science/pith/Q673SXFMAXHU5HRT6EKSAGSILK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Q673SXFMAXHU5HRT6EKSAGSILK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T00:25:49Z","links":{"resolver":"https://pith.science/pith/Q673SXFMAXHU5HRT6EKSAGSILK","bundle":"https://pith.science/pith/Q673SXFMAXHU5HRT6EKSAGSILK/bundle.json","state":"https://pith.science/pith/Q673SXFMAXHU5HRT6EKSAGSILK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Q673SXFMAXHU5HRT6EKSAGSILK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2026:Q673SXFMAXHU5HRT6EKSAGSILK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"7f538b77cc93d783609f35dd6f255e5dad1ca91e1e4506e56d1312b30b6fd0b4","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-07-15T13:28:18Z","title_canon_sha256":"03ea9ecea1b2ed4cfcf95d0b84f75d248853b6f23f4e39d54189b250e23868ba"},"schema_version":"1.0","source":{"id":"2607.13820","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2607.13820","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"arxiv_version","alias_value":"2607.13820v1","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2607.13820","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"pith_short_12","alias_value":"Q673SXFMAXHU","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"pith_short_16","alias_value":"Q673SXFMAXHU5HRT","created_at":"2026-07-16T01:23:08Z"},{"alias_kind":"pith_short_8","alias_value":"Q673SXFM","created_at":"2026-07-16T01:23:08Z"}],"graph_snapshots":[{"event_id":"sha256:ca59d2cf684a32318166642b7aaea56fe025865325f694d12a9488bc2df3b256","target":"graph","created_at":"2026-07-16T01:23:08Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2607.13820/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) are increasingly being used in everyday software engineering tasks, particularly in automated code generation. Despite their widespread adoption, these models remain far from perfect, making systematic and fair evaluation essential to understand their strengths and limitations. In the context of code generation, existing benchmarks are limited: they often target a single programming language and rely primarily on unit test outcomes, while overlooking other critical dimensions such as the overall quality of the generated code and its closeness to a valid solution. T","authors_text":"Jo\\~ao R. Campos, Marco Vieira, Rodrigo Pato Nogueira","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-07-15T13:28:18Z","title":"PROBE: Benchmarking Code Generation in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2607.13820","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8982bfe910200e0aa0b3d693016a57b0e1c22bb1eedf712fb7ab062bdf54c502","target":"record","created_at":"2026-07-16T01:23:08Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"7f538b77cc93d783609f35dd6f255e5dad1ca91e1e4506e56d1312b30b6fd0b4","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2026-07-15T13:28:18Z","title_canon_sha256":"03ea9ecea1b2ed4cfcf95d0b84f75d248853b6f23f4e39d54189b250e23868ba"},"schema_version":"1.0","source":{"id":"2607.13820","kind":"arxiv","version":1}},"canonical_sha256":"87bfb95cac05cf4e9e33f115201a485a8d1b03c4b41ace4d9577bcb8900520a5","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"87bfb95cac05cf4e9e33f115201a485a8d1b03c4b41ace4d9577bcb8900520a5","first_computed_at":"2026-07-16T01:23:08.094900Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-16T01:23:08.094900Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"TcgFu9IXlwNuTZ7hYgOqASHpCqidriFLVpyrJKaoZzvEePscfxfGj74Jl33o7cuoJutu9prvn472H0R0K2v7Bg==","signature_status":"signed_v1","signed_at":"2026-07-16T01:23:08.095706Z","signed_message":"canonical_sha256_bytes"},"source_id":"2607.13820","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8982bfe910200e0aa0b3d693016a57b0e1c22bb1eedf712fb7ab062bdf54c502","sha256:ca59d2cf684a32318166642b7aaea56fe025865325f694d12a9488bc2df3b256"],"state_sha256":"9b552d2a5d1a2025d01f0f8f9c762d464b463e40ee4079c73863333932e8bcd9"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hkqvP1dZDRuha8LSG6DzRtS/u9pqeQ3eOb9L+rvLC+qcrWBSql7pTTKnPnrTeux94Nmy+Vli92MpMeCr3ePoCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T00:25:49.313161Z","bundle_sha256":"c6b593de46f4cce7c5921350491e82a35983074931075af058da310d8478a083"}}