{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:GZ7SVRRZT63YGOXPF3EOYR3Y4D","short_pith_number":"pith:GZ7SVRRZ","schema_version":"1.0","canonical_sha256":"367f2ac6399fb7833aef2ec8ec4778e0f720dc8edb54de6e692c5ed0868699a7","source":{"kind":"arxiv","id":"2409.07440","version":1},"attestation_state":"computed","paper":{"title":"SUPER: Evaluating Agents on Setting Up and Executing Tasks from Research Repositories","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.SE"],"primary_cat":"cs.AI","authors_text":"Ashish Sabharwal, Ben Bogin, Erin Bransom, Kejuan Yang, Kyle Richardson, Peter Clark, Shashank Gupta, Tushar Khot","submitted_at":"2024-09-11T17:37:48Z","abstract_excerpt":"Given that Large Language Models (LLMs) have made significant progress in writing code, can they now be used to autonomously reproduce results from research repositories? Such a capability would be a boon to the research community, helping researchers validate, understand, and extend prior work. To advance towards this goal, we introduce SUPER, the first benchmark designed to evaluate the capability of LLMs in setting up and executing tasks from research repositories. SUPERaims to capture the realistic challenges faced by researchers working with Machine Learning (ML) and Natural Language Proc"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2409.07440","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-09-11T17:37:48Z","cross_cats_sorted":["cs.CL","cs.SE"],"title_canon_sha256":"25c9f016616e81161045104fe6c00b4299402b402972bcd23f0c7547ef0b71c5","abstract_canon_sha256":"56a9aed895ec96714bc73f0bc269447244d020a4f2918f50bacb9971cb77026f"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:05:57.430552Z","signature_b64":"gKgcKQWsk7yRSfTmced783iUQAQ5sHkwtkPi48OlfDRNwso7dcpgqZH9GyTpcq8GyfxXfM4llTu2RpBdGRCRCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"367f2ac6399fb7833aef2ec8ec4778e0f720dc8edb54de6e692c5ed0868699a7","last_reissued_at":"2026-07-05T09:05:57.430114Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:05:57.430114Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"SUPER: Evaluating Agents on Setting Up and Executing Tasks from Research Repositories","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.SE"],"primary_cat":"cs.AI","authors_text":"Ashish Sabharwal, Ben Bogin, Erin Bransom, Kejuan Yang, Kyle Richardson, Peter Clark, Shashank Gupta, Tushar Khot","submitted_at":"2024-09-11T17:37:48Z","abstract_excerpt":"Given that Large Language Models (LLMs) have made significant progress in writing code, can they now be used to autonomously reproduce results from research repositories? Such a capability would be a boon to the research community, helping researchers validate, understand, and extend prior work. To advance towards this goal, we introduce SUPER, the first benchmark designed to evaluate the capability of LLMs in setting up and executing tasks from research repositories. SUPERaims to capture the realistic challenges faced by researchers working with Machine Learning (ML) and Natural Language Proc"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.07440","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.07440/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2409.07440","created_at":"2026-07-05T09:05:57.430176+00:00"},{"alias_kind":"arxiv_version","alias_value":"2409.07440v1","created_at":"2026-07-05T09:05:57.430176+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.07440","created_at":"2026-07-05T09:05:57.430176+00:00"},{"alias_kind":"pith_short_12","alias_value":"GZ7SVRRZT63Y","created_at":"2026-07-05T09:05:57.430176+00:00"},{"alias_kind":"pith_short_16","alias_value":"GZ7SVRRZT63YGOXP","created_at":"2026-07-05T09:05:57.430176+00:00"},{"alias_kind":"pith_short_8","alias_value":"GZ7SVRRZ","created_at":"2026-07-05T09:05:57.430176+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":4,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2509.20374","citing_title":"CFDLLMBench: A Benchmark Suite for Evaluating Large Language Models in Computational Fluid Dynamics","ref_index":9,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12808","citing_title":"Neurodata Without Boredom: Benchmarking Agentic AI for Data Reuse","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2605.12808","citing_title":"Neurodata Without Boredom: Benchmarking Agentic AI for Data Reuse","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2604.11270","citing_title":"Evaluating LLM Agents on Automated Software Analysis Tasks","ref_index":4,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D","json":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D.json","graph_json":"https://pith.science/api/pith-number/GZ7SVRRZT63YGOXPF3EOYR3Y4D/graph.json","events_json":"https://pith.science/api/pith-number/GZ7SVRRZT63YGOXPF3EOYR3Y4D/events.json","paper":"https://pith.science/paper/GZ7SVRRZ"},"agent_actions":{"view_html":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D","download_json":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D.json","view_paper":"https://pith.science/paper/GZ7SVRRZ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2409.07440&json=true","fetch_graph":"https://pith.science/api/pith-number/GZ7SVRRZT63YGOXPF3EOYR3Y4D/graph.json","fetch_events":"https://pith.science/api/pith-number/GZ7SVRRZT63YGOXPF3EOYR3Y4D/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/action/timestamp_anchor","attest_storage":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/action/storage_attestation","attest_author":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/action/author_attestation","sign_citation":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/action/citation_signature","submit_replication":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/action/replication_record"}},"created_at":"2026-07-05T09:05:57.430176+00:00","updated_at":"2026-07-05T09:05:57.430176+00:00"}