{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:DD6MWZHP5Q47Z3WVZ5BAKCAUHE","short_pith_number":"pith:DD6MWZHP","canonical_record":{"source":{"id":"2502.00226","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-01-31T23:47:02Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"e1dba20308b8161fd60945051c0ccdcef18f204af565f3b496ff46e0274b6cc2","abstract_canon_sha256":"5938f45ba0e5dc73974178641e3a1a968bcfef2e886054d9d76d8154078d82fc"},"schema_version":"1.0"},"canonical_sha256":"18fccb64efec39fceed5cf420508143934ee6ff627530fbc29e20d1eb19be661","source":{"kind":"arxiv","id":"2502.00226","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.00226","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"arxiv_version","alias_value":"2502.00226v1","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.00226","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"pith_short_12","alias_value":"DD6MWZHP5Q47","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"pith_short_16","alias_value":"DD6MWZHP5Q47Z3WV","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"pith_short_8","alias_value":"DD6MWZHP","created_at":"2026-07-05T10:08:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:DD6MWZHP5Q47Z3WVZ5BAKCAUHE","target":"record","payload":{"canonical_record":{"source":{"id":"2502.00226","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-01-31T23:47:02Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"e1dba20308b8161fd60945051c0ccdcef18f204af565f3b496ff46e0274b6cc2","abstract_canon_sha256":"5938f45ba0e5dc73974178641e3a1a968bcfef2e886054d9d76d8154078d82fc"},"schema_version":"1.0"},"canonical_sha256":"18fccb64efec39fceed5cf420508143934ee6ff627530fbc29e20d1eb19be661","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:08:30.791480Z","signature_b64":"wGNFQgL/0at0Xdluu7UVO4k1PC74nw1MDiavmlqjMXhQ0V+sqf0b50Wrr1YzXDaXk72ZdSZG3GdZIF99J+o3Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"18fccb64efec39fceed5cf420508143934ee6ff627530fbc29e20d1eb19be661","last_reissued_at":"2026-07-05T10:08:30.790973Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:08:30.790973Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.00226","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:08:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"O8xNtsc4rFtnZ+MP0aOW17TJQ6icSKw+zcrLOExuYMETpPVzxYfoGM29Lagoap8q9VdT5X+WcflRffIX4rP/Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T23:46:03.821346Z"},"content_sha256":"4c4133d7fb9958b065383acd1c89f48d7e6ca1f62184b53b15da59d088f8165c","schema_version":"1.0","event_id":"sha256:4c4133d7fb9958b065383acd1c89f48d7e6ca1f62184b53b15da59d088f8165c"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:DD6MWZHP5Q47Z3WVZ5BAKCAUHE","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"HackerRank-ASTRA: Evaluating Correctness & Consistency of Large Language Models on cross-domain multi-file project problems","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","headline":"","cross_cats":["cs.SE"],"primary_cat":"cs.LG","authors_text":"Darshan Suresh, Jun Xing, Mayur Bhatia, Rafik Matta, Sahil Phulwani","submitted_at":"2025-01-31T23:47:02Z","abstract_excerpt":"Evaluating the real-world applicability of large language models (LLMs) provides valuable insights for their development and use in software development tasks. Existing benchmarks often focus on standalone coding problems or specific libraries, overlooking multi-file, project-based scenarios and lacking a rigorous evaluation of consistency. The HackerRank-ASTRA Benchmark introduces project-based coding problems that mirror real-world scenarios. It evaluates model consistency through 32 runs (k = 32) and median standard deviation while incorporating taxonomy-level analysis to assess sub-skill c"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.00226","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.00226/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:08:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"preTWoZ5At8tVA6/+BknqWhsStd5MPlIopYBXqL/WQNuw5Dzkk9uW1zctBnBZIW/hDFJWAwMSIyZ+sAbXV/5DQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T23:46:03.821880Z"},"content_sha256":"96e474e6f1946c5d31821714e7f6058fe1c366561ac6416edf441e3c7ba057ae","schema_version":"1.0","event_id":"sha256:96e474e6f1946c5d31821714e7f6058fe1c366561ac6416edf441e3c7ba057ae"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DD6MWZHP5Q47Z3WVZ5BAKCAUHE/bundle.json","state_url":"https://pith.science/pith/DD6MWZHP5Q47Z3WVZ5BAKCAUHE/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DD6MWZHP5Q47Z3WVZ5BAKCAUHE/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-13T23:46:03Z","links":{"resolver":"https://pith.science/pith/DD6MWZHP5Q47Z3WVZ5BAKCAUHE","bundle":"https://pith.science/pith/DD6MWZHP5Q47Z3WVZ5BAKCAUHE/bundle.json","state":"https://pith.science/pith/DD6MWZHP5Q47Z3WVZ5BAKCAUHE/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DD6MWZHP5Q47Z3WVZ5BAKCAUHE/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:DD6MWZHP5Q47Z3WVZ5BAKCAUHE","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5938f45ba0e5dc73974178641e3a1a968bcfef2e886054d9d76d8154078d82fc","cross_cats_sorted":["cs.SE"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-01-31T23:47:02Z","title_canon_sha256":"e1dba20308b8161fd60945051c0ccdcef18f204af565f3b496ff46e0274b6cc2"},"schema_version":"1.0","source":{"id":"2502.00226","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.00226","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"arxiv_version","alias_value":"2502.00226v1","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.00226","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"pith_short_12","alias_value":"DD6MWZHP5Q47","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"pith_short_16","alias_value":"DD6MWZHP5Q47Z3WV","created_at":"2026-07-05T10:08:30Z"},{"alias_kind":"pith_short_8","alias_value":"DD6MWZHP","created_at":"2026-07-05T10:08:30Z"}],"graph_snapshots":[{"event_id":"sha256:96e474e6f1946c5d31821714e7f6058fe1c366561ac6416edf441e3c7ba057ae","target":"graph","created_at":"2026-07-05T10:08:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.00226/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Evaluating the real-world applicability of large language models (LLMs) provides valuable insights for their development and use in software development tasks. Existing benchmarks often focus on standalone coding problems or specific libraries, overlooking multi-file, project-based scenarios and lacking a rigorous evaluation of consistency. The HackerRank-ASTRA Benchmark introduces project-based coding problems that mirror real-world scenarios. It evaluates model consistency through 32 runs (k = 32) and median standard deviation while incorporating taxonomy-level analysis to assess sub-skill c","authors_text":"Darshan Suresh, Jun Xing, Mayur Bhatia, Rafik Matta, Sahil Phulwani","cross_cats":["cs.SE"],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-01-31T23:47:02Z","title":"HackerRank-ASTRA: Evaluating Correctness & Consistency of Large Language Models on cross-domain multi-file project problems"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.00226","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4c4133d7fb9958b065383acd1c89f48d7e6ca1f62184b53b15da59d088f8165c","target":"record","created_at":"2026-07-05T10:08:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5938f45ba0e5dc73974178641e3a1a968bcfef2e886054d9d76d8154078d82fc","cross_cats_sorted":["cs.SE"],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.LG","submitted_at":"2025-01-31T23:47:02Z","title_canon_sha256":"e1dba20308b8161fd60945051c0ccdcef18f204af565f3b496ff46e0274b6cc2"},"schema_version":"1.0","source":{"id":"2502.00226","kind":"arxiv","version":1}},"canonical_sha256":"18fccb64efec39fceed5cf420508143934ee6ff627530fbc29e20d1eb19be661","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"18fccb64efec39fceed5cf420508143934ee6ff627530fbc29e20d1eb19be661","first_computed_at":"2026-07-05T10:08:30.790973Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:08:30.790973Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"wGNFQgL/0at0Xdluu7UVO4k1PC74nw1MDiavmlqjMXhQ0V+sqf0b50Wrr1YzXDaXk72ZdSZG3GdZIF99J+o3Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T10:08:30.791480Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.00226","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4c4133d7fb9958b065383acd1c89f48d7e6ca1f62184b53b15da59d088f8165c","sha256:96e474e6f1946c5d31821714e7f6058fe1c366561ac6416edf441e3c7ba057ae"],"state_sha256":"24da5dca5029dff32306b43bd049c58df41bdddeec56b8c86bf260549bc44d57"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"BH0G9ncvP096G4SaSjIt9byr5La6y/qUCjdd/vTkO5cyhsvWyjTdVLPuCGVLNKXUP/zlF8Nqwk+WN6kQeQL/AA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-13T23:46:03.843076Z","bundle_sha256":"18fc80020c7322d5bfb75ccad26278ff47c15424fae7b347b4e5e0cf1848f6c9"}}