{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:YLIE675ST757U3COLGXSBFQHVO","short_pith_number":"pith:YLIE675S","canonical_record":{"source":{"id":"2407.13168","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-18T05:15:24Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"0f0d4af86e4bde915a80c933df34891d1d66c412d4070022871266e89db08849","abstract_canon_sha256":"385954f62f77d5c444d036e651ae25b3ac0984e3a0d4151afe88a4a9560311a4"},"schema_version":"1.0"},"canonical_sha256":"c2d04f7fb29ffbfa6c4e59af209607ab98cd17ce0d23fd482ed22eb94421370e","source":{"kind":"arxiv","id":"2407.13168","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.13168","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"arxiv_version","alias_value":"2407.13168v1","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.13168","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"pith_short_12","alias_value":"YLIE675ST757","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"pith_short_16","alias_value":"YLIE675ST757U3CO","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"pith_short_8","alias_value":"YLIE675S","created_at":"2026-07-05T08:45:35Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:YLIE675ST757U3COLGXSBFQHVO","target":"record","payload":{"canonical_record":{"source":{"id":"2407.13168","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-18T05:15:24Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"0f0d4af86e4bde915a80c933df34891d1d66c412d4070022871266e89db08849","abstract_canon_sha256":"385954f62f77d5c444d036e651ae25b3ac0984e3a0d4151afe88a4a9560311a4"},"schema_version":"1.0"},"canonical_sha256":"c2d04f7fb29ffbfa6c4e59af209607ab98cd17ce0d23fd482ed22eb94421370e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:45:35.052277Z","signature_b64":"h4pO5cfq50XSe76zsRHlmhnj0lSeWWJt858JvGnaHo4GIZfANK+6liow6P3Gy9BqSe923oCaRh88Dc/91jK2Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"c2d04f7fb29ffbfa6c4e59af209607ab98cd17ce0d23fd482ed22eb94421370e","last_reissued_at":"2026-07-05T08:45:35.051808Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:45:35.051808Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.13168","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:45:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FYAD6jOg42zhMb/uCwl+18wz88nAemRdI7I7FJcrq5SAdO20F18gAYMxxFcP/2qSdcZ2bzKDs4L5TvIFKM9wAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T17:18:45.696789Z"},"content_sha256":"b01ace6de26ad6f4c32311d12781d396441fa347b535bd61a4f827b0aa3bf1eb","schema_version":"1.0","event_id":"sha256:b01ace6de26ad6f4c32311d12781d396441fa347b535bd61a4f827b0aa3bf1eb"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:YLIE675ST757U3COLGXSBFQHVO","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SciCode: A Research Coding Benchmark Curated by Scientists","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.AI","authors_text":"Bohao Wu, Chenyu Tian, Cunwei Fan, Di Luo, Eliu Huerta, Genglin Liu, Hao Peng, Hao Tong, Jamie Callan, Kha Trinh, Kilian Lieret, Kittithat Krongchon, Luyu Gao, Minhui Zhu, Minyang Tian, Ofir Press, Pan Ji, Roland Haas, Shengyan Liu, Shengzhu Yin, Shizhuo Dylan Zhang, Tianhua Tao, Xinan Chen, Xuefei Guo, Yanxin Lu, Yanyu Xiong, Yao Li, Yufeng Du, Yutao ma, Zihan Wang","submitted_at":"2024-07-18T05:15:24Z","abstract_excerpt":"Since language models (LMs) now outperform average humans on many challenging tasks, it has become increasingly difficult to develop challenging, high-quality, and realistic evaluations. We address this issue by examining LMs' capabilities to generate code for solving real scientific research problems. Incorporating input from scientists and AI researchers in 16 diverse natural science sub-fields, including mathematics, physics, chemistry, biology, and materials science, we created a scientist-curated coding benchmark, SciCode. The problems in SciCode naturally factorize into multiple subprobl"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.13168","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.13168/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:45:35Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"6ROrGTFs1sDOPAAI5r2JebPlH6eJ+yBWTY9nO1R86aDY+GX8pSm6L3ovAGa1ia9KwUFKfMPT7/Hp0Wr2I0XtCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T17:18:45.697307Z"},"content_sha256":"f1830daa27325380b4132963c3d2be8cb095d47204572dcd5d42c036fc09ba00","schema_version":"1.0","event_id":"sha256:f1830daa27325380b4132963c3d2be8cb095d47204572dcd5d42c036fc09ba00"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/YLIE675ST757U3COLGXSBFQHVO/bundle.json","state_url":"https://pith.science/pith/YLIE675ST757U3COLGXSBFQHVO/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/YLIE675ST757U3COLGXSBFQHVO/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T17:18:45Z","links":{"resolver":"https://pith.science/pith/YLIE675ST757U3COLGXSBFQHVO","bundle":"https://pith.science/pith/YLIE675ST757U3COLGXSBFQHVO/bundle.json","state":"https://pith.science/pith/YLIE675ST757U3COLGXSBFQHVO/state.json","well_known_bundle":"https://pith.science/.well-known/pith/YLIE675ST757U3COLGXSBFQHVO/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:YLIE675ST757U3COLGXSBFQHVO","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"385954f62f77d5c444d036e651ae25b3ac0984e3a0d4151afe88a4a9560311a4","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-18T05:15:24Z","title_canon_sha256":"0f0d4af86e4bde915a80c933df34891d1d66c412d4070022871266e89db08849"},"schema_version":"1.0","source":{"id":"2407.13168","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.13168","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"arxiv_version","alias_value":"2407.13168v1","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.13168","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"pith_short_12","alias_value":"YLIE675ST757","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"pith_short_16","alias_value":"YLIE675ST757U3CO","created_at":"2026-07-05T08:45:35Z"},{"alias_kind":"pith_short_8","alias_value":"YLIE675S","created_at":"2026-07-05T08:45:35Z"}],"graph_snapshots":[{"event_id":"sha256:f1830daa27325380b4132963c3d2be8cb095d47204572dcd5d42c036fc09ba00","target":"graph","created_at":"2026-07-05T08:45:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.13168/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Since language models (LMs) now outperform average humans on many challenging tasks, it has become increasingly difficult to develop challenging, high-quality, and realistic evaluations. We address this issue by examining LMs' capabilities to generate code for solving real scientific research problems. Incorporating input from scientists and AI researchers in 16 diverse natural science sub-fields, including mathematics, physics, chemistry, biology, and materials science, we created a scientist-curated coding benchmark, SciCode. The problems in SciCode naturally factorize into multiple subprobl","authors_text":"Bohao Wu, Chenyu Tian, Cunwei Fan, Di Luo, Eliu Huerta, Genglin Liu, Hao Peng, Hao Tong, Jamie Callan, Kha Trinh, Kilian Lieret, Kittithat Krongchon, Luyu Gao, Minhui Zhu, Minyang Tian, Ofir Press, Pan Ji, Roland Haas, Shengyan Liu, Shengzhu Yin, Shizhuo Dylan Zhang, Tianhua Tao, Xinan Chen, Xuefei Guo, Yanxin Lu, Yanyu Xiong, Yao Li, Yufeng Du, Yutao ma, Zihan Wang","cross_cats":["cs.CL"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-18T05:15:24Z","title":"SciCode: A Research Coding Benchmark Curated by Scientists"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.13168","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b01ace6de26ad6f4c32311d12781d396441fa347b535bd61a4f827b0aa3bf1eb","target":"record","created_at":"2026-07-05T08:45:35Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"385954f62f77d5c444d036e651ae25b3ac0984e3a0d4151afe88a4a9560311a4","cross_cats_sorted":["cs.CL"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-07-18T05:15:24Z","title_canon_sha256":"0f0d4af86e4bde915a80c933df34891d1d66c412d4070022871266e89db08849"},"schema_version":"1.0","source":{"id":"2407.13168","kind":"arxiv","version":1}},"canonical_sha256":"c2d04f7fb29ffbfa6c4e59af209607ab98cd17ce0d23fd482ed22eb94421370e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"c2d04f7fb29ffbfa6c4e59af209607ab98cd17ce0d23fd482ed22eb94421370e","first_computed_at":"2026-07-05T08:45:35.051808Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:45:35.051808Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"h4pO5cfq50XSe76zsRHlmhnj0lSeWWJt858JvGnaHo4GIZfANK+6liow6P3Gy9BqSe923oCaRh88Dc/91jK2Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:45:35.052277Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.13168","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b01ace6de26ad6f4c32311d12781d396441fa347b535bd61a4f827b0aa3bf1eb","sha256:f1830daa27325380b4132963c3d2be8cb095d47204572dcd5d42c036fc09ba00"],"state_sha256":"f9562bfb3a2ba0d68818d3a8cd785c04d5a6ebcbc615680babae8aa3d36a3827"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"eynm9Zu+AUZ4i8JodEB47UhdFMmyf0pkx1WdL4yEyaq5Rc6hmurQcfjqlwOaCRNi0MlWrTeuRA2MTWVR3B5jDw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T17:18:45.702169Z","bundle_sha256":"f3eb6a196ba96592062f8929798c7505ff882459ffc094f86117b47db26e1997"}}