{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:JQDGEE2VDX6CZD3Y5XLUCJEPSF","short_pith_number":"pith:JQDGEE2V","canonical_record":{"source":{"id":"2502.00561","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CY","submitted_at":"2025-02-01T21:09:51Z","cross_cats_sorted":[],"title_canon_sha256":"782aa48e08e58ad0e7432375ae6fbb712128ef92b8ce93889a7b4978e436c6ca","abstract_canon_sha256":"c4e66f9e34438b578c345efd7b8656e69cef34f59320226bbaa310846915232a"},"schema_version":"1.0"},"canonical_sha256":"4c066213551dfc2c8f78edd741248f9147d20981dd8aa8d69396ff3af995b937","source":{"kind":"arxiv","id":"2502.00561","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.00561","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"arxiv_version","alias_value":"2502.00561v2","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.00561","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"pith_short_12","alias_value":"JQDGEE2VDX6C","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"pith_short_16","alias_value":"JQDGEE2VDX6CZD3Y","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"pith_short_8","alias_value":"JQDGEE2V","created_at":"2026-07-05T11:17:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:JQDGEE2VDX6CZD3Y5XLUCJEPSF","target":"record","payload":{"canonical_record":{"source":{"id":"2502.00561","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CY","submitted_at":"2025-02-01T21:09:51Z","cross_cats_sorted":[],"title_canon_sha256":"782aa48e08e58ad0e7432375ae6fbb712128ef92b8ce93889a7b4978e436c6ca","abstract_canon_sha256":"c4e66f9e34438b578c345efd7b8656e69cef34f59320226bbaa310846915232a"},"schema_version":"1.0"},"canonical_sha256":"4c066213551dfc2c8f78edd741248f9147d20981dd8aa8d69396ff3af995b937","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:17:40.326461Z","signature_b64":"ObM5f4o2NY/5huXmeS2989hCKOkwYIwpFwlpICRo6dEUlZ5C4tphqG/C8qXMVDuf0IrVEIFSeLtyrIEDBVf9AQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4c066213551dfc2c8f78edd741248f9147d20981dd8aa8d69396ff3af995b937","last_reissued_at":"2026-07-05T11:17:40.325960Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:17:40.325960Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.00561","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:17:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"z81a117wnIPPPlTSZxJ/dF+MIQVjcPgemRDlNTDJZL7H/BTbV5pcxsDj0geQBzFORSyQkIVMYPX8XYbzdHhCBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T08:18:04.453773Z"},"content_sha256":"20233b720535f3f2b49ade68bcfc7673070d7f0e929d247f932a154913b624b0","schema_version":"1.0","event_id":"sha256:20233b720535f3f2b49ade68bcfc7673070d7f0e929d247f932a154913b624b0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:JQDGEE2VDX6CZD3Y5XLUCJEPSF","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Position: Evaluating Generative AI Systems Is a Social Science Measurement Challenge","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CY","authors_text":"Abigail Z. Jacobs, A. Feder Cooper, Alexandra Chouldechova, Alexandra Olteanu, Angelina Wang, Chad Atalla, Dan Vann, Emily Corvi, Emily Sheng, Hannah Washington, Hanna Wallach, Jean Garcia-Gathright, Jennifer Wortman Vaughan, Matthew Vogel, Meera Desai, Nicholas Pangakis, P. Alex Dow, Solon Barocas, Stefanie Reed, Su Lin Blodgett","submitted_at":"2025-02-01T21:09:51Z","abstract_excerpt":"The measurement tasks involved in evaluating generative AI (GenAI) systems lack sufficient scientific rigor, leading to what has been described as \"a tangle of sloppy tests [and] apples-to-oranges comparisons\" (Roose, 2024). In this position paper, we argue that the ML community would benefit from learning from and drawing on the social sciences when developing and using measurement instruments for evaluating GenAI systems. Specifically, our position is that evaluating GenAI systems is a social science measurement challenge. We present a four-level framework, grounded in measurement theory fro"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.00561","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.00561/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:17:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PhZ3voR3LR5Ok+DlEaKnusmKXFaPjmwOGRpWczAMy6wl7LxFBA/ARFUDjsKqot3wtPj1C4F8z1muJ5qJK2lVCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T08:18:04.454271Z"},"content_sha256":"93fed1fdd5159e683ec20dcb8067a6f156fd0226c84821b8ed98a3eb4a290d5e","schema_version":"1.0","event_id":"sha256:93fed1fdd5159e683ec20dcb8067a6f156fd0226c84821b8ed98a3eb4a290d5e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/JQDGEE2VDX6CZD3Y5XLUCJEPSF/bundle.json","state_url":"https://pith.science/pith/JQDGEE2VDX6CZD3Y5XLUCJEPSF/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/JQDGEE2VDX6CZD3Y5XLUCJEPSF/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T08:18:04Z","links":{"resolver":"https://pith.science/pith/JQDGEE2VDX6CZD3Y5XLUCJEPSF","bundle":"https://pith.science/pith/JQDGEE2VDX6CZD3Y5XLUCJEPSF/bundle.json","state":"https://pith.science/pith/JQDGEE2VDX6CZD3Y5XLUCJEPSF/state.json","well_known_bundle":"https://pith.science/.well-known/pith/JQDGEE2VDX6CZD3Y5XLUCJEPSF/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:JQDGEE2VDX6CZD3Y5XLUCJEPSF","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c4e66f9e34438b578c345efd7b8656e69cef34f59320226bbaa310846915232a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CY","submitted_at":"2025-02-01T21:09:51Z","title_canon_sha256":"782aa48e08e58ad0e7432375ae6fbb712128ef92b8ce93889a7b4978e436c6ca"},"schema_version":"1.0","source":{"id":"2502.00561","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.00561","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"arxiv_version","alias_value":"2502.00561v2","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.00561","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"pith_short_12","alias_value":"JQDGEE2VDX6C","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"pith_short_16","alias_value":"JQDGEE2VDX6CZD3Y","created_at":"2026-07-05T11:17:40Z"},{"alias_kind":"pith_short_8","alias_value":"JQDGEE2V","created_at":"2026-07-05T11:17:40Z"}],"graph_snapshots":[{"event_id":"sha256:93fed1fdd5159e683ec20dcb8067a6f156fd0226c84821b8ed98a3eb4a290d5e","target":"graph","created_at":"2026-07-05T11:17:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.00561/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The measurement tasks involved in evaluating generative AI (GenAI) systems lack sufficient scientific rigor, leading to what has been described as \"a tangle of sloppy tests [and] apples-to-oranges comparisons\" (Roose, 2024). In this position paper, we argue that the ML community would benefit from learning from and drawing on the social sciences when developing and using measurement instruments for evaluating GenAI systems. Specifically, our position is that evaluating GenAI systems is a social science measurement challenge. We present a four-level framework, grounded in measurement theory fro","authors_text":"Abigail Z. Jacobs, A. Feder Cooper, Alexandra Chouldechova, Alexandra Olteanu, Angelina Wang, Chad Atalla, Dan Vann, Emily Corvi, Emily Sheng, Hannah Washington, Hanna Wallach, Jean Garcia-Gathright, Jennifer Wortman Vaughan, Matthew Vogel, Meera Desai, Nicholas Pangakis, P. Alex Dow, Solon Barocas, Stefanie Reed, Su Lin Blodgett","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CY","submitted_at":"2025-02-01T21:09:51Z","title":"Position: Evaluating Generative AI Systems Is a Social Science Measurement Challenge"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.00561","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:20233b720535f3f2b49ade68bcfc7673070d7f0e929d247f932a154913b624b0","target":"record","created_at":"2026-07-05T11:17:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c4e66f9e34438b578c345efd7b8656e69cef34f59320226bbaa310846915232a","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CY","submitted_at":"2025-02-01T21:09:51Z","title_canon_sha256":"782aa48e08e58ad0e7432375ae6fbb712128ef92b8ce93889a7b4978e436c6ca"},"schema_version":"1.0","source":{"id":"2502.00561","kind":"arxiv","version":2}},"canonical_sha256":"4c066213551dfc2c8f78edd741248f9147d20981dd8aa8d69396ff3af995b937","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4c066213551dfc2c8f78edd741248f9147d20981dd8aa8d69396ff3af995b937","first_computed_at":"2026-07-05T11:17:40.325960Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:17:40.325960Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ObM5f4o2NY/5huXmeS2989hCKOkwYIwpFwlpICRo6dEUlZ5C4tphqG/C8qXMVDuf0IrVEIFSeLtyrIEDBVf9AQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:17:40.326461Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.00561","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:20233b720535f3f2b49ade68bcfc7673070d7f0e929d247f932a154913b624b0","sha256:93fed1fdd5159e683ec20dcb8067a6f156fd0226c84821b8ed98a3eb4a290d5e"],"state_sha256":"6f34835e9aa4cdfa58a34e5628c84a523b74196af4d1c41f7516510019aba28e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FKZZrRnRmYpF46QEGqsXJTgko4x0z7WeAEc/O9vaQ43XCH23CV2SGsQGVPckr1e4wTJGhlc8FyEml3KKPa2rBQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T08:18:04.458550Z","bundle_sha256":"4b4e55a8c4756586584106efeea54edd1b434f85830ba7ef985ba7ba8050b655"}}