{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:5UU5FHVVPQIVV2R42MNMXMIV2Z","short_pith_number":"pith:5UU5FHVV","canonical_record":{"source":{"id":"2302.04434","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-02-09T04:43:10Z","cross_cats_sorted":["cs.AI","cs.HC","cs.LG"],"title_canon_sha256":"e9ee1fdb45ef08db5d88c678deeec45214291840f00de5d7007a85036ef7029d","abstract_canon_sha256":"76fb4e85d35cf3cc2eab2be9051c4c028c3e136a36529267b22153983e19ac07"},"schema_version":"1.0"},"canonical_sha256":"ed29d29eb57c115aea3cd31acbb115d6586111f773c6994656d682ff2e598283","source":{"kind":"arxiv","id":"2302.04434","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2302.04434","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"arxiv_version","alias_value":"2302.04434v1","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2302.04434","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"pith_short_12","alias_value":"5UU5FHVVPQIV","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"pith_short_16","alias_value":"5UU5FHVVPQIVV2R4","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"pith_short_8","alias_value":"5UU5FHVV","created_at":"2026-07-05T05:40:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:5UU5FHVVPQIVV2R42MNMXMIV2Z","target":"record","payload":{"canonical_record":{"source":{"id":"2302.04434","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-02-09T04:43:10Z","cross_cats_sorted":["cs.AI","cs.HC","cs.LG"],"title_canon_sha256":"e9ee1fdb45ef08db5d88c678deeec45214291840f00de5d7007a85036ef7029d","abstract_canon_sha256":"76fb4e85d35cf3cc2eab2be9051c4c028c3e136a36529267b22153983e19ac07"},"schema_version":"1.0"},"canonical_sha256":"ed29d29eb57c115aea3cd31acbb115d6586111f773c6994656d682ff2e598283","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:40:13.779441Z","signature_b64":"h/sAjmCpF3Fy2BrfbhUmAbcN+7qLMpo3kwikYPfRCKypHlZlg7ovXT8FjKf4QRZ/z5IFlRHN0ympkh9doaF6Cg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ed29d29eb57c115aea3cd31acbb115d6586111f773c6994656d682ff2e598283","last_reissued_at":"2026-07-05T05:40:13.778972Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:40:13.778972Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2302.04434","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:40:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Jma0U5ErRnqtZAclTe7HHIId6H2Sy/Lcq6PIqHiII9vwMienDe06h8IIDMXZizVFAeiDXSVfwQZAjC7JHfWXCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-28T00:54:50.903775Z"},"content_sha256":"9f13904c6d2854b7a66cb99dd016c1ddb9f98be132046eedfd17e8fa8e47b46b","schema_version":"1.0","event_id":"sha256:9f13904c6d2854b7a66cb99dd016c1ddb9f98be132046eedfd17e8fa8e47b46b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:5UU5FHVVPQIVV2R42MNMXMIV2Z","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Real-Time Visual Feedback to Guide Benchmark Creation: A Human-and-Metric-in-the-Loop Workflow","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.HC","cs.LG"],"primary_cat":"cs.CL","authors_text":"Anjana Arunkumar, Bhavdeep Sachdeva, Chitta Baral, Chris Bryan, Swaroop Mishra","submitted_at":"2023-02-09T04:43:10Z","abstract_excerpt":"Recent research has shown that language models exploit `artifacts' in benchmarks to solve tasks, rather than truly learning them, leading to inflated model performance. In pursuit of creating better benchmarks, we propose VAIDA, a novel benchmark creation paradigm for NLP, that focuses on guiding crowdworkers, an under-explored facet of addressing benchmark idiosyncrasies. VAIDA facilitates sample correction by providing realtime visual feedback and recommendations to improve sample quality. Our approach is domain, model, task, and metric agnostic, and constitutes a paradigm shift for robust, "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2302.04434","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2302.04434/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:40:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"rwoDV0qAcIMEmEYW8YMFWl71mWFPeOiOiB1WOYKGJVTaE9N8JOJWNzEUUHkGaDbcRP3nae5SvmdSrU9kphW5BQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-28T00:54:50.904163Z"},"content_sha256":"bc062a9faa8c20f3577a1ffc5d6813e9604a0a4dd781950ca4e2643c23546ddb","schema_version":"1.0","event_id":"sha256:bc062a9faa8c20f3577a1ffc5d6813e9604a0a4dd781950ca4e2643c23546ddb"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/5UU5FHVVPQIVV2R42MNMXMIV2Z/bundle.json","state_url":"https://pith.science/pith/5UU5FHVVPQIVV2R42MNMXMIV2Z/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/5UU5FHVVPQIVV2R42MNMXMIV2Z/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-28T00:54:50Z","links":{"resolver":"https://pith.science/pith/5UU5FHVVPQIVV2R42MNMXMIV2Z","bundle":"https://pith.science/pith/5UU5FHVVPQIVV2R42MNMXMIV2Z/bundle.json","state":"https://pith.science/pith/5UU5FHVVPQIVV2R42MNMXMIV2Z/state.json","well_known_bundle":"https://pith.science/.well-known/pith/5UU5FHVVPQIVV2R42MNMXMIV2Z/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:5UU5FHVVPQIVV2R42MNMXMIV2Z","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"76fb4e85d35cf3cc2eab2be9051c4c028c3e136a36529267b22153983e19ac07","cross_cats_sorted":["cs.AI","cs.HC","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-02-09T04:43:10Z","title_canon_sha256":"e9ee1fdb45ef08db5d88c678deeec45214291840f00de5d7007a85036ef7029d"},"schema_version":"1.0","source":{"id":"2302.04434","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2302.04434","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"arxiv_version","alias_value":"2302.04434v1","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2302.04434","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"pith_short_12","alias_value":"5UU5FHVVPQIV","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"pith_short_16","alias_value":"5UU5FHVVPQIVV2R4","created_at":"2026-07-05T05:40:13Z"},{"alias_kind":"pith_short_8","alias_value":"5UU5FHVV","created_at":"2026-07-05T05:40:13Z"}],"graph_snapshots":[{"event_id":"sha256:bc062a9faa8c20f3577a1ffc5d6813e9604a0a4dd781950ca4e2643c23546ddb","target":"graph","created_at":"2026-07-05T05:40:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2302.04434/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent research has shown that language models exploit `artifacts' in benchmarks to solve tasks, rather than truly learning them, leading to inflated model performance. In pursuit of creating better benchmarks, we propose VAIDA, a novel benchmark creation paradigm for NLP, that focuses on guiding crowdworkers, an under-explored facet of addressing benchmark idiosyncrasies. VAIDA facilitates sample correction by providing realtime visual feedback and recommendations to improve sample quality. Our approach is domain, model, task, and metric agnostic, and constitutes a paradigm shift for robust, ","authors_text":"Anjana Arunkumar, Bhavdeep Sachdeva, Chitta Baral, Chris Bryan, Swaroop Mishra","cross_cats":["cs.AI","cs.HC","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-02-09T04:43:10Z","title":"Real-Time Visual Feedback to Guide Benchmark Creation: A Human-and-Metric-in-the-Loop Workflow"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2302.04434","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:9f13904c6d2854b7a66cb99dd016c1ddb9f98be132046eedfd17e8fa8e47b46b","target":"record","created_at":"2026-07-05T05:40:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"76fb4e85d35cf3cc2eab2be9051c4c028c3e136a36529267b22153983e19ac07","cross_cats_sorted":["cs.AI","cs.HC","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-02-09T04:43:10Z","title_canon_sha256":"e9ee1fdb45ef08db5d88c678deeec45214291840f00de5d7007a85036ef7029d"},"schema_version":"1.0","source":{"id":"2302.04434","kind":"arxiv","version":1}},"canonical_sha256":"ed29d29eb57c115aea3cd31acbb115d6586111f773c6994656d682ff2e598283","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ed29d29eb57c115aea3cd31acbb115d6586111f773c6994656d682ff2e598283","first_computed_at":"2026-07-05T05:40:13.778972Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:40:13.778972Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"h/sAjmCpF3Fy2BrfbhUmAbcN+7qLMpo3kwikYPfRCKypHlZlg7ovXT8FjKf4QRZ/z5IFlRHN0ympkh9doaF6Cg==","signature_status":"signed_v1","signed_at":"2026-07-05T05:40:13.779441Z","signed_message":"canonical_sha256_bytes"},"source_id":"2302.04434","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:9f13904c6d2854b7a66cb99dd016c1ddb9f98be132046eedfd17e8fa8e47b46b","sha256:bc062a9faa8c20f3577a1ffc5d6813e9604a0a4dd781950ca4e2643c23546ddb"],"state_sha256":"bc0c6090b86cadb912868006747ec63efbe85e1543eadaa41e8ed380bea8bc94"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nDdwxe3yon/X5nOfaaHY2GY3Bw1uRpl7YxlQSFfIAsxDVEeVgma+WQyfIwaflO0MgWGgnWOUgYjjh7/6GRX1Bg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-28T00:54:50.906429Z","bundle_sha256":"70d3429b83787da079c603e51c80b10f1ce1d4e75f66376f1d128ca0da8fa460"}}