{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:GZ7SVRRZT63YGOXPF3EOYR3Y4D","short_pith_number":"pith:GZ7SVRRZ","canonical_record":{"source":{"id":"2409.07440","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-09-11T17:37:48Z","cross_cats_sorted":["cs.CL","cs.SE"],"title_canon_sha256":"25c9f016616e81161045104fe6c00b4299402b402972bcd23f0c7547ef0b71c5","abstract_canon_sha256":"56a9aed895ec96714bc73f0bc269447244d020a4f2918f50bacb9971cb77026f"},"schema_version":"1.0"},"canonical_sha256":"367f2ac6399fb7833aef2ec8ec4778e0f720dc8edb54de6e692c5ed0868699a7","source":{"kind":"arxiv","id":"2409.07440","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.07440","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"arxiv_version","alias_value":"2409.07440v1","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.07440","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"pith_short_12","alias_value":"GZ7SVRRZT63Y","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"pith_short_16","alias_value":"GZ7SVRRZT63YGOXP","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"pith_short_8","alias_value":"GZ7SVRRZ","created_at":"2026-07-05T09:05:57Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:GZ7SVRRZT63YGOXPF3EOYR3Y4D","target":"record","payload":{"canonical_record":{"source":{"id":"2409.07440","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-09-11T17:37:48Z","cross_cats_sorted":["cs.CL","cs.SE"],"title_canon_sha256":"25c9f016616e81161045104fe6c00b4299402b402972bcd23f0c7547ef0b71c5","abstract_canon_sha256":"56a9aed895ec96714bc73f0bc269447244d020a4f2918f50bacb9971cb77026f"},"schema_version":"1.0"},"canonical_sha256":"367f2ac6399fb7833aef2ec8ec4778e0f720dc8edb54de6e692c5ed0868699a7","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:05:57.430552Z","signature_b64":"gKgcKQWsk7yRSfTmced783iUQAQ5sHkwtkPi48OlfDRNwso7dcpgqZH9GyTpcq8GyfxXfM4llTu2RpBdGRCRCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"367f2ac6399fb7833aef2ec8ec4778e0f720dc8edb54de6e692c5ed0868699a7","last_reissued_at":"2026-07-05T09:05:57.430114Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:05:57.430114Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.07440","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:05:57Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5mMRX1RAZvL944h/FlFLqWioHhJaMvMA/cUghgox1abxiF7qGnr1dV1kFpOcJ/xGAZaJNYGQKxIS1b6iNDhqCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T00:09:03.690634Z"},"content_sha256":"43c143774d1cc30c03383f7afee1f9e01a7045512c9576d11f39e32ec3708988","schema_version":"1.0","event_id":"sha256:43c143774d1cc30c03383f7afee1f9e01a7045512c9576d11f39e32ec3708988"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:GZ7SVRRZT63YGOXPF3EOYR3Y4D","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SUPER: Evaluating Agents on Setting Up and Executing Tasks from Research Repositories","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.SE"],"primary_cat":"cs.AI","authors_text":"Ashish Sabharwal, Ben Bogin, Erin Bransom, Kejuan Yang, Kyle Richardson, Peter Clark, Shashank Gupta, Tushar Khot","submitted_at":"2024-09-11T17:37:48Z","abstract_excerpt":"Given that Large Language Models (LLMs) have made significant progress in writing code, can they now be used to autonomously reproduce results from research repositories? Such a capability would be a boon to the research community, helping researchers validate, understand, and extend prior work. To advance towards this goal, we introduce SUPER, the first benchmark designed to evaluate the capability of LLMs in setting up and executing tasks from research repositories. SUPERaims to capture the realistic challenges faced by researchers working with Machine Learning (ML) and Natural Language Proc"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.07440","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.07440/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:05:57Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"H7QUoSroMsEcQw2ceF+kpikZOWyVFD8O1RXPIvetYCawPFspcODW5MGgG6PS7i8D0uePgOwPK3zx6wRw6MOLDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T00:09:03.691007Z"},"content_sha256":"24076fb1aa7e86d60eddcdba84edc592da2d45eb6284096fee98e81223c2d255","schema_version":"1.0","event_id":"sha256:24076fb1aa7e86d60eddcdba84edc592da2d45eb6284096fee98e81223c2d255"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/bundle.json","state_url":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T00:09:03Z","links":{"resolver":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D","bundle":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/bundle.json","state":"https://pith.science/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GZ7SVRRZT63YGOXPF3EOYR3Y4D/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GZ7SVRRZT63YGOXPF3EOYR3Y4D","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"56a9aed895ec96714bc73f0bc269447244d020a4f2918f50bacb9971cb77026f","cross_cats_sorted":["cs.CL","cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-09-11T17:37:48Z","title_canon_sha256":"25c9f016616e81161045104fe6c00b4299402b402972bcd23f0c7547ef0b71c5"},"schema_version":"1.0","source":{"id":"2409.07440","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.07440","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"arxiv_version","alias_value":"2409.07440v1","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.07440","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"pith_short_12","alias_value":"GZ7SVRRZT63Y","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"pith_short_16","alias_value":"GZ7SVRRZT63YGOXP","created_at":"2026-07-05T09:05:57Z"},{"alias_kind":"pith_short_8","alias_value":"GZ7SVRRZ","created_at":"2026-07-05T09:05:57Z"}],"graph_snapshots":[{"event_id":"sha256:24076fb1aa7e86d60eddcdba84edc592da2d45eb6284096fee98e81223c2d255","target":"graph","created_at":"2026-07-05T09:05:57Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.07440/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Given that Large Language Models (LLMs) have made significant progress in writing code, can they now be used to autonomously reproduce results from research repositories? Such a capability would be a boon to the research community, helping researchers validate, understand, and extend prior work. To advance towards this goal, we introduce SUPER, the first benchmark designed to evaluate the capability of LLMs in setting up and executing tasks from research repositories. SUPERaims to capture the realistic challenges faced by researchers working with Machine Learning (ML) and Natural Language Proc","authors_text":"Ashish Sabharwal, Ben Bogin, Erin Bransom, Kejuan Yang, Kyle Richardson, Peter Clark, Shashank Gupta, Tushar Khot","cross_cats":["cs.CL","cs.SE"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-09-11T17:37:48Z","title":"SUPER: Evaluating Agents on Setting Up and Executing Tasks from Research Repositories"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.07440","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:43c143774d1cc30c03383f7afee1f9e01a7045512c9576d11f39e32ec3708988","target":"record","created_at":"2026-07-05T09:05:57Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"56a9aed895ec96714bc73f0bc269447244d020a4f2918f50bacb9971cb77026f","cross_cats_sorted":["cs.CL","cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.AI","submitted_at":"2024-09-11T17:37:48Z","title_canon_sha256":"25c9f016616e81161045104fe6c00b4299402b402972bcd23f0c7547ef0b71c5"},"schema_version":"1.0","source":{"id":"2409.07440","kind":"arxiv","version":1}},"canonical_sha256":"367f2ac6399fb7833aef2ec8ec4778e0f720dc8edb54de6e692c5ed0868699a7","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"367f2ac6399fb7833aef2ec8ec4778e0f720dc8edb54de6e692c5ed0868699a7","first_computed_at":"2026-07-05T09:05:57.430114Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:05:57.430114Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gKgcKQWsk7yRSfTmced783iUQAQ5sHkwtkPi48OlfDRNwso7dcpgqZH9GyTpcq8GyfxXfM4llTu2RpBdGRCRCg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:05:57.430552Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.07440","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:43c143774d1cc30c03383f7afee1f9e01a7045512c9576d11f39e32ec3708988","sha256:24076fb1aa7e86d60eddcdba84edc592da2d45eb6284096fee98e81223c2d255"],"state_sha256":"160e06a1bdc11743e410ad751e1c5da965cfffa23770d16079ebc7e99ec45838"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"SUYi2f/AWJWBIjb7UziRdETlTHNjeYKrwf47yynk127yzB83fPIqcA5dIl++ImZuOkcwLLeaSu0cbmWXNJbfDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T00:09:03.693019Z","bundle_sha256":"5fd6d6dc5408c98165dd0a7d8810ab15d93c5dcb96ea4f4922a33b7821fdab9d"}}