{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:I7R2ORFJJS2MTPLW4MU4YAAJS7","short_pith_number":"pith:I7R2ORFJ","canonical_record":{"source":{"id":"2311.09835","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-16T12:03:21Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"d852bbc0aead1ab4c321944655dab59321ac7bc3894c8148e4ac8e8c1fa35585","abstract_canon_sha256":"1445b578891a6698e4f704b2ab8c63ddb62a09ce00385ac4c007fd65516d61cf"},"schema_version":"1.0"},"canonical_sha256":"47e3a744a94cb4c9bd76e329cc000997c78e6c16db887ef51f2d10e8cb522125","source":{"kind":"arxiv","id":"2311.09835","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.09835","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"arxiv_version","alias_value":"2311.09835v5","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.09835","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"pith_short_12","alias_value":"I7R2ORFJJS2M","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"pith_short_16","alias_value":"I7R2ORFJJS2MTPLW","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"pith_short_8","alias_value":"I7R2ORFJ","created_at":"2026-07-05T08:57:34Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:I7R2ORFJJS2MTPLW4MU4YAAJS7","target":"record","payload":{"canonical_record":{"source":{"id":"2311.09835","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-16T12:03:21Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"d852bbc0aead1ab4c321944655dab59321ac7bc3894c8148e4ac8e8c1fa35585","abstract_canon_sha256":"1445b578891a6698e4f704b2ab8c63ddb62a09ce00385ac4c007fd65516d61cf"},"schema_version":"1.0"},"canonical_sha256":"47e3a744a94cb4c9bd76e329cc000997c78e6c16db887ef51f2d10e8cb522125","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:57:34.750087Z","signature_b64":"qswhFVwlIULo1kozB96L52FYHrgRLddT3LjO4shu90ATqXLX/7O6NV+HcaRdIV8v6dPY2b15VKBuSsnEuUNkCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"47e3a744a94cb4c9bd76e329cc000997c78e6c16db887ef51f2d10e8cb522125","last_reissued_at":"2026-07-05T08:57:34.749564Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:57:34.749564Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2311.09835","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:57:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dkzNCuFjq0dctMj6BpQLHdJJnxCm5J6/jTFsyt4FzCHODTVTkFaS/8lCIl91PwjhK7c5fFoxlMTlhNnR52isBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T20:34:52.761730Z"},"content_sha256":"8f359f244f307e6cc0be30a48e6e5d07240a1cf842328634f1ea37f1fc106f93","schema_version":"1.0","event_id":"sha256:8f359f244f307e6cc0be30a48e6e5d07240a1cf842328634f1ea37f1fc106f93"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:I7R2ORFJJS2MTPLW4MU4YAAJS7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Arman Cohan, Baobao Chang, Haozhe Zhao, Helan hu, Junjie Lu, Kaikai An, Liang Chen, Mark Gerstein, Ruijun Huang, Sheng Chen, Shuzheng Si, Tianyu Liu, Wangchunshu Zhou, Xiangru Tang, Yanjun Shao, Yan Wang, Yichi Zhang, Yilun Zhao, Yin Fang, Yujia Qin, Yuliang Liu, Zefan Cai, Zexuan Deng, Zhiwei Jiang","submitted_at":"2023-11-16T12:03:21Z","abstract_excerpt":"Despite Large Language Models (LLMs) like GPT-4 achieving impressive results in function-level code generation, they struggle with repository-scale code understanding (e.g., coming up with the right arguments for calling routines), requiring a deeper comprehension of complex file interactions. Also, recently, people have developed LLM agents that attempt to interact with repository code (e.g., compiling and evaluating its execution), prompting the need to evaluate their performance. These gaps have motivated our development of ML-Bench, a benchmark rooted in real-world programming applications"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.09835","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2311.09835/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:57:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4bPTbJSkk3ZVilHrm53Ibi3XRh89VtcypP90rejP6krbbArXGhogIfg/72I1bqhJ8H9PlOYMtLFFOF424maKAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-08T20:34:52.762115Z"},"content_sha256":"1601f33af5316f1a3bc22435771072f053a9d4a1b08711443135daf76bac41a8","schema_version":"1.0","event_id":"sha256:1601f33af5316f1a3bc22435771072f053a9d4a1b08711443135daf76bac41a8"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/bundle.json","state_url":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-08T20:34:52Z","links":{"resolver":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7","bundle":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/bundle.json","state":"https://pith.science/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/I7R2ORFJJS2MTPLW4MU4YAAJS7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:I7R2ORFJJS2MTPLW4MU4YAAJS7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"1445b578891a6698e4f704b2ab8c63ddb62a09ce00385ac4c007fd65516d61cf","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-16T12:03:21Z","title_canon_sha256":"d852bbc0aead1ab4c321944655dab59321ac7bc3894c8148e4ac8e8c1fa35585"},"schema_version":"1.0","source":{"id":"2311.09835","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2311.09835","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"arxiv_version","alias_value":"2311.09835v5","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2311.09835","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"pith_short_12","alias_value":"I7R2ORFJJS2M","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"pith_short_16","alias_value":"I7R2ORFJJS2MTPLW","created_at":"2026-07-05T08:57:34Z"},{"alias_kind":"pith_short_8","alias_value":"I7R2ORFJ","created_at":"2026-07-05T08:57:34Z"}],"graph_snapshots":[{"event_id":"sha256:1601f33af5316f1a3bc22435771072f053a9d4a1b08711443135daf76bac41a8","target":"graph","created_at":"2026-07-05T08:57:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2311.09835/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Despite Large Language Models (LLMs) like GPT-4 achieving impressive results in function-level code generation, they struggle with repository-scale code understanding (e.g., coming up with the right arguments for calling routines), requiring a deeper comprehension of complex file interactions. Also, recently, people have developed LLM agents that attempt to interact with repository code (e.g., compiling and evaluating its execution), prompting the need to evaluate their performance. These gaps have motivated our development of ML-Bench, a benchmark rooted in real-world programming applications","authors_text":"Arman Cohan, Baobao Chang, Haozhe Zhao, Helan hu, Junjie Lu, Kaikai An, Liang Chen, Mark Gerstein, Ruijun Huang, Sheng Chen, Shuzheng Si, Tianyu Liu, Wangchunshu Zhou, Xiangru Tang, Yanjun Shao, Yan Wang, Yichi Zhang, Yilun Zhao, Yin Fang, Yujia Qin, Yuliang Liu, Zefan Cai, Zexuan Deng, Zhiwei Jiang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-16T12:03:21Z","title":"ML-Bench: Evaluating Large Language Models and Agents for Machine Learning Tasks on Repository-Level Code"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2311.09835","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:8f359f244f307e6cc0be30a48e6e5d07240a1cf842328634f1ea37f1fc106f93","target":"record","created_at":"2026-07-05T08:57:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"1445b578891a6698e4f704b2ab8c63ddb62a09ce00385ac4c007fd65516d61cf","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-11-16T12:03:21Z","title_canon_sha256":"d852bbc0aead1ab4c321944655dab59321ac7bc3894c8148e4ac8e8c1fa35585"},"schema_version":"1.0","source":{"id":"2311.09835","kind":"arxiv","version":5}},"canonical_sha256":"47e3a744a94cb4c9bd76e329cc000997c78e6c16db887ef51f2d10e8cb522125","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"47e3a744a94cb4c9bd76e329cc000997c78e6c16db887ef51f2d10e8cb522125","first_computed_at":"2026-07-05T08:57:34.749564Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:57:34.749564Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"qswhFVwlIULo1kozB96L52FYHrgRLddT3LjO4shu90ATqXLX/7O6NV+HcaRdIV8v6dPY2b15VKBuSsnEuUNkCg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:57:34.750087Z","signed_message":"canonical_sha256_bytes"},"source_id":"2311.09835","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:8f359f244f307e6cc0be30a48e6e5d07240a1cf842328634f1ea37f1fc106f93","sha256:1601f33af5316f1a3bc22435771072f053a9d4a1b08711443135daf76bac41a8"],"state_sha256":"e487ebc913cf5bb66f6d878cc81d2c97a40213d746f2f98378f6f36eed0b5269"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vjU2zJlZ7B0AJCrIarxs5NghXHcxKmn8V7R5YCrKyFqTqd5ww0uqsglk31pXcCM61/yFmIUNHM8+r8IFCvTWDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-08T20:34:52.764409Z","bundle_sha256":"5a4db4df1fb6ee9858007372454483958dadd58bcf90b7d2ca72d33f97c1af0d"}}