{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:DKNY3Q2LTZYUVKBE2J2RQGMRN5","short_pith_number":"pith:DKNY3Q2L","canonical_record":{"source":{"id":"2410.24005","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:06:16Z","cross_cats_sorted":[],"title_canon_sha256":"40e143d23419311483b015d54c6aaf0d49b6088aa8546428c8d4d419feec4f48","abstract_canon_sha256":"8a251fa1e9ced64b8c9182e3bca855b898da4eba9c9073f47e8d8c0c12165d3b"},"schema_version":"1.0"},"canonical_sha256":"1a9b8dc34b9e714aa824d2751819916f6aac43a17ab3f86af198aa79bdc00cd8","source":{"kind":"arxiv","id":"2410.24005","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.24005","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"arxiv_version","alias_value":"2410.24005v1","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.24005","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"pith_short_12","alias_value":"DKNY3Q2LTZYU","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"pith_short_16","alias_value":"DKNY3Q2LTZYUVKBE","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"pith_short_8","alias_value":"DKNY3Q2L","created_at":"2026-07-05T09:29:17Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:DKNY3Q2LTZYUVKBE2J2RQGMRN5","target":"record","payload":{"canonical_record":{"source":{"id":"2410.24005","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:06:16Z","cross_cats_sorted":[],"title_canon_sha256":"40e143d23419311483b015d54c6aaf0d49b6088aa8546428c8d4d419feec4f48","abstract_canon_sha256":"8a251fa1e9ced64b8c9182e3bca855b898da4eba9c9073f47e8d8c0c12165d3b"},"schema_version":"1.0"},"canonical_sha256":"1a9b8dc34b9e714aa824d2751819916f6aac43a17ab3f86af198aa79bdc00cd8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:29:17.763948Z","signature_b64":"LWVI8FwgdF6pMzQGzlc/k3rKLGICFF17Hr+FNgvDCLDQfDuj3fzhdJTCWP4vFyuQOVq2faZTcWgTepv0wegyDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1a9b8dc34b9e714aa824d2751819916f6aac43a17ab3f86af198aa79bdc00cd8","last_reissued_at":"2026-07-05T09:29:17.763360Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:29:17.763360Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2410.24005","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:29:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"k3CMv6PtRywKfYwzq15Xa5sW0y6MVft4YRVfgj5q8IVyy0itc6H5I/0D5uLc0BFkqMtIf2Uts82BKJUo+afrBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T11:38:46.635299Z"},"content_sha256":"4fc9385e56e47bc9ed7987998ec6e8aeb564b22371a4a4c9df409f318650fb04","schema_version":"1.0","event_id":"sha256:4fc9385e56e47bc9ed7987998ec6e8aeb564b22371a4a4c9df409f318650fb04"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:DKNY3Q2LTZYUVKBE2J2RQGMRN5","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Context-Aware Testing: A New Paradigm for Model Testing with Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.LG","authors_text":"Max Ruiz Luyten, Mihaela van der Schaar, Nabeel Seedat, Paulius Rauba","submitted_at":"2024-10-31T15:06:16Z","abstract_excerpt":"The predominant de facto paradigm of testing ML models relies on either using only held-out data to compute aggregate evaluation metrics or by assessing the performance on different subgroups. However, such data-only testing methods operate under the restrictive assumption that the available empirical data is the sole input for testing ML models, disregarding valuable contextual information that could guide model testing. In this paper, we challenge the go-to approach of data-only testing and introduce context-aware testing (CAT) which uses context as an inductive bias to guide the search for "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.24005","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2410.24005/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:29:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nkN3A6J4Zhjk1hHGUKUgqUzprGO5SLmfJBK04bCiekJLde3PEcp4dBlB9kKbHQ6rY6dqfICzubsNR7CdLBnADA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T11:38:46.636076Z"},"content_sha256":"d05ecc95291053516cf1f4a88ad7bf06636d4b71055e71b3c4641100357e24eb","schema_version":"1.0","event_id":"sha256:d05ecc95291053516cf1f4a88ad7bf06636d4b71055e71b3c4641100357e24eb"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/DKNY3Q2LTZYUVKBE2J2RQGMRN5/bundle.json","state_url":"https://pith.science/pith/DKNY3Q2LTZYUVKBE2J2RQGMRN5/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/DKNY3Q2LTZYUVKBE2J2RQGMRN5/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T11:38:46Z","links":{"resolver":"https://pith.science/pith/DKNY3Q2LTZYUVKBE2J2RQGMRN5","bundle":"https://pith.science/pith/DKNY3Q2LTZYUVKBE2J2RQGMRN5/bundle.json","state":"https://pith.science/pith/DKNY3Q2LTZYUVKBE2J2RQGMRN5/state.json","well_known_bundle":"https://pith.science/.well-known/pith/DKNY3Q2LTZYUVKBE2J2RQGMRN5/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:DKNY3Q2LTZYUVKBE2J2RQGMRN5","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"8a251fa1e9ced64b8c9182e3bca855b898da4eba9c9073f47e8d8c0c12165d3b","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:06:16Z","title_canon_sha256":"40e143d23419311483b015d54c6aaf0d49b6088aa8546428c8d4d419feec4f48"},"schema_version":"1.0","source":{"id":"2410.24005","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2410.24005","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"arxiv_version","alias_value":"2410.24005v1","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2410.24005","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"pith_short_12","alias_value":"DKNY3Q2LTZYU","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"pith_short_16","alias_value":"DKNY3Q2LTZYUVKBE","created_at":"2026-07-05T09:29:17Z"},{"alias_kind":"pith_short_8","alias_value":"DKNY3Q2L","created_at":"2026-07-05T09:29:17Z"}],"graph_snapshots":[{"event_id":"sha256:d05ecc95291053516cf1f4a88ad7bf06636d4b71055e71b3c4641100357e24eb","target":"graph","created_at":"2026-07-05T09:29:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2410.24005/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The predominant de facto paradigm of testing ML models relies on either using only held-out data to compute aggregate evaluation metrics or by assessing the performance on different subgroups. However, such data-only testing methods operate under the restrictive assumption that the available empirical data is the sole input for testing ML models, disregarding valuable contextual information that could guide model testing. In this paper, we challenge the go-to approach of data-only testing and introduce context-aware testing (CAT) which uses context as an inductive bias to guide the search for ","authors_text":"Max Ruiz Luyten, Mihaela van der Schaar, Nabeel Seedat, Paulius Rauba","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:06:16Z","title":"Context-Aware Testing: A New Paradigm for Model Testing with Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2410.24005","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:4fc9385e56e47bc9ed7987998ec6e8aeb564b22371a4a4c9df409f318650fb04","target":"record","created_at":"2026-07-05T09:29:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"8a251fa1e9ced64b8c9182e3bca855b898da4eba9c9073f47e8d8c0c12165d3b","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.LG","submitted_at":"2024-10-31T15:06:16Z","title_canon_sha256":"40e143d23419311483b015d54c6aaf0d49b6088aa8546428c8d4d419feec4f48"},"schema_version":"1.0","source":{"id":"2410.24005","kind":"arxiv","version":1}},"canonical_sha256":"1a9b8dc34b9e714aa824d2751819916f6aac43a17ab3f86af198aa79bdc00cd8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"1a9b8dc34b9e714aa824d2751819916f6aac43a17ab3f86af198aa79bdc00cd8","first_computed_at":"2026-07-05T09:29:17.763360Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:29:17.763360Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"LWVI8FwgdF6pMzQGzlc/k3rKLGICFF17Hr+FNgvDCLDQfDuj3fzhdJTCWP4vFyuQOVq2faZTcWgTepv0wegyDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:29:17.763948Z","signed_message":"canonical_sha256_bytes"},"source_id":"2410.24005","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:4fc9385e56e47bc9ed7987998ec6e8aeb564b22371a4a4c9df409f318650fb04","sha256:d05ecc95291053516cf1f4a88ad7bf06636d4b71055e71b3c4641100357e24eb"],"state_sha256":"b162ba8af5403e33bc50aa442a6d5b00bcbaab189612b1f16ad54469d3177e18"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3UI/Z5NmmXsCH56KYPWSkw+1336sOS9NXA8BX5flnnABH2zrhiaD4GFn5+ixV5OF4tOI9vby+vI36YRrqgt6Cw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T11:38:46.640903Z","bundle_sha256":"85e3127b42cc861efbfd500676326c2a2e377a1ef25477e92767362609734569"}}