{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:IML5I7R52WYSTG6JCH6HHJBZ4P","short_pith_number":"pith:IML5I7R5","canonical_record":{"source":{"id":"2504.12211","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-04-16T15:58:33Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"3f19a92fd042be1cad643519526b6f933e5b22c1ffdb73b64f4c1c98b3e4941f","abstract_canon_sha256":"9090395da64ddec4abd980108cdf9ec9d92c3eda85848fbb9dac20a4fe4a5f69"},"schema_version":"1.0"},"canonical_sha256":"4317d47e3dd5b1299bc911fc73a439e3dddd1b9d0e4170ea7ba7f2287602a962","source":{"kind":"arxiv","id":"2504.12211","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.12211","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"arxiv_version","alias_value":"2504.12211v1","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.12211","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"pith_short_12","alias_value":"IML5I7R52WYS","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"pith_short_16","alias_value":"IML5I7R52WYSTG6J","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"pith_short_8","alias_value":"IML5I7R5","created_at":"2026-07-05T10:50:04Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:IML5I7R52WYSTG6JCH6HHJBZ4P","target":"record","payload":{"canonical_record":{"source":{"id":"2504.12211","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-04-16T15:58:33Z","cross_cats_sorted":["cs.HC"],"title_canon_sha256":"3f19a92fd042be1cad643519526b6f933e5b22c1ffdb73b64f4c1c98b3e4941f","abstract_canon_sha256":"9090395da64ddec4abd980108cdf9ec9d92c3eda85848fbb9dac20a4fe4a5f69"},"schema_version":"1.0"},"canonical_sha256":"4317d47e3dd5b1299bc911fc73a439e3dddd1b9d0e4170ea7ba7f2287602a962","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:50:04.261190Z","signature_b64":"E7NAaP0+X8PXby5BiGijISEx59mWWGhhEGKdBo5gxmWGkja6TRMd24lHS4/ssVhxmy+zjinekn31S6EPO2iRAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"4317d47e3dd5b1299bc911fc73a439e3dddd1b9d0e4170ea7ba7f2287602a962","last_reissued_at":"2026-07-05T10:50:04.260719Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:50:04.260719Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2504.12211","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:50:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Xrn844tj/HNSkTEaeUBij+PUss3k1PmUD3E3Ss2IYUlFzjNJ2ig03k94LJRWqq1ek6dnvF5A3TgbRK+YKz/FDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-26T18:50:10.612502Z"},"content_sha256":"5152575b7c7773e9caef28f431e03a57732947b76e5ff4881f1e5df8b09b43b0","schema_version":"1.0","event_id":"sha256:5152575b7c7773e9caef28f431e03a57732947b76e5ff4881f1e5df8b09b43b0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:IML5I7R52WYSTG6JCH6HHJBZ4P","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Creating benchmarkable components to measure the quality ofAI-enhanced developer tools","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.HC"],"primary_cat":"cs.SE","authors_text":"Ambar Murillo, Andrew Macvean, Ben Ferrari-Church, Elise Paradis, Matthew Hughes, Maulishree Pandey, Sarah D'Angelo","submitted_at":"2025-04-16T15:58:33Z","abstract_excerpt":"In the AI community, benchmarks to evaluate model quality are well established, but an equivalent approach to benchmarking products built upon generative AI models is still missing. This has had two consequences. First, it has made teams focus on model quality over the developer experience, while successful products combine both. Second, product team have struggled to answer questions about their products in relation to their competitors.\n  In this case study, we share: (1) our process to create robust, enterprise-grade and modular components to support the benchmarking of the developer experi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.12211","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2504.12211/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:50:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Sw+vkLjcXBEA8eevCxvBkkxTZz6G8YhQL0KKslKEZntHU0K4ubD3kRurm3ML4e4PKGZTpP852tT2xFvwz5rvDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-26T18:50:10.612870Z"},"content_sha256":"de0bc9fb34d612e25dc508c524803ce7ddb1b5b20f60b47e9abff6cb86ce8742","schema_version":"1.0","event_id":"sha256:de0bc9fb34d612e25dc508c524803ce7ddb1b5b20f60b47e9abff6cb86ce8742"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/IML5I7R52WYSTG6JCH6HHJBZ4P/bundle.json","state_url":"https://pith.science/pith/IML5I7R52WYSTG6JCH6HHJBZ4P/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/IML5I7R52WYSTG6JCH6HHJBZ4P/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-26T18:50:10Z","links":{"resolver":"https://pith.science/pith/IML5I7R52WYSTG6JCH6HHJBZ4P","bundle":"https://pith.science/pith/IML5I7R52WYSTG6JCH6HHJBZ4P/bundle.json","state":"https://pith.science/pith/IML5I7R52WYSTG6JCH6HHJBZ4P/state.json","well_known_bundle":"https://pith.science/.well-known/pith/IML5I7R52WYSTG6JCH6HHJBZ4P/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:IML5I7R52WYSTG6JCH6HHJBZ4P","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"9090395da64ddec4abd980108cdf9ec9d92c3eda85848fbb9dac20a4fe4a5f69","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-04-16T15:58:33Z","title_canon_sha256":"3f19a92fd042be1cad643519526b6f933e5b22c1ffdb73b64f4c1c98b3e4941f"},"schema_version":"1.0","source":{"id":"2504.12211","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2504.12211","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"arxiv_version","alias_value":"2504.12211v1","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2504.12211","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"pith_short_12","alias_value":"IML5I7R52WYS","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"pith_short_16","alias_value":"IML5I7R52WYSTG6J","created_at":"2026-07-05T10:50:04Z"},{"alias_kind":"pith_short_8","alias_value":"IML5I7R5","created_at":"2026-07-05T10:50:04Z"}],"graph_snapshots":[{"event_id":"sha256:de0bc9fb34d612e25dc508c524803ce7ddb1b5b20f60b47e9abff6cb86ce8742","target":"graph","created_at":"2026-07-05T10:50:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2504.12211/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In the AI community, benchmarks to evaluate model quality are well established, but an equivalent approach to benchmarking products built upon generative AI models is still missing. This has had two consequences. First, it has made teams focus on model quality over the developer experience, while successful products combine both. Second, product team have struggled to answer questions about their products in relation to their competitors.\n  In this case study, we share: (1) our process to create robust, enterprise-grade and modular components to support the benchmarking of the developer experi","authors_text":"Ambar Murillo, Andrew Macvean, Ben Ferrari-Church, Elise Paradis, Matthew Hughes, Maulishree Pandey, Sarah D'Angelo","cross_cats":["cs.HC"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-04-16T15:58:33Z","title":"Creating benchmarkable components to measure the quality ofAI-enhanced developer tools"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2504.12211","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5152575b7c7773e9caef28f431e03a57732947b76e5ff4881f1e5df8b09b43b0","target":"record","created_at":"2026-07-05T10:50:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"9090395da64ddec4abd980108cdf9ec9d92c3eda85848fbb9dac20a4fe4a5f69","cross_cats_sorted":["cs.HC"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.SE","submitted_at":"2025-04-16T15:58:33Z","title_canon_sha256":"3f19a92fd042be1cad643519526b6f933e5b22c1ffdb73b64f4c1c98b3e4941f"},"schema_version":"1.0","source":{"id":"2504.12211","kind":"arxiv","version":1}},"canonical_sha256":"4317d47e3dd5b1299bc911fc73a439e3dddd1b9d0e4170ea7ba7f2287602a962","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"4317d47e3dd5b1299bc911fc73a439e3dddd1b9d0e4170ea7ba7f2287602a962","first_computed_at":"2026-07-05T10:50:04.260719Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:50:04.260719Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"E7NAaP0+X8PXby5BiGijISEx59mWWGhhEGKdBo5gxmWGkja6TRMd24lHS4/ssVhxmy+zjinekn31S6EPO2iRAA==","signature_status":"signed_v1","signed_at":"2026-07-05T10:50:04.261190Z","signed_message":"canonical_sha256_bytes"},"source_id":"2504.12211","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5152575b7c7773e9caef28f431e03a57732947b76e5ff4881f1e5df8b09b43b0","sha256:de0bc9fb34d612e25dc508c524803ce7ddb1b5b20f60b47e9abff6cb86ce8742"],"state_sha256":"79caf668c818b3b1befe055cfcf82a92e675520e9c982ce6166c7f477c51c001"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"H8fEax9vOcL10CHp2kh3kCUd4ACnXx5m/btQc40+D98C2caBzxr2R1bdxeh+YDTem+mkNDEoWWtyTfgF8dSjCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-26T18:50:10.614930Z","bundle_sha256":"35e29fdebaf221c69bef72b2cbe7d119b1e837f0fdf2c0e9874cc7c877f18033"}}