{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:QZ7IJYD3RC665ECICHB7P2AKMV","short_pith_number":"pith:QZ7IJYD3","canonical_record":{"source":{"id":"2009.05015","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.AP","submitted_at":"2020-09-10T17:28:14Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"c3aa34dab0f947b0814e8e1a215a36e09a0588c7c729e9bbe8ae78e1b12e82ec","abstract_canon_sha256":"5b11b41b633c20077215bc380656703bb17019a0cc7c64b6c2bea360e769c06a"},"schema_version":"1.0"},"canonical_sha256":"867e84e07b88bdee904811c3f7e80a6555911156d30722c07f3c6d3e8cb1f34c","source":{"kind":"arxiv","id":"2009.05015","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2009.05015","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"arxiv_version","alias_value":"2009.05015v1","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2009.05015","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"pith_short_12","alias_value":"QZ7IJYD3RC66","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"pith_short_16","alias_value":"QZ7IJYD3RC665ECI","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"pith_short_8","alias_value":"QZ7IJYD3","created_at":"2026-07-05T01:34:31Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:QZ7IJYD3RC665ECICHB7P2AKMV","target":"record","payload":{"canonical_record":{"source":{"id":"2009.05015","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.AP","submitted_at":"2020-09-10T17:28:14Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"c3aa34dab0f947b0814e8e1a215a36e09a0588c7c729e9bbe8ae78e1b12e82ec","abstract_canon_sha256":"5b11b41b633c20077215bc380656703bb17019a0cc7c64b6c2bea360e769c06a"},"schema_version":"1.0"},"canonical_sha256":"867e84e07b88bdee904811c3f7e80a6555911156d30722c07f3c6d3e8cb1f34c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:34:31.974834Z","signature_b64":"gU0onY6D1nh4tBG5W+wo1gN2tnx2G109osVqf0+QwMTyHSW2x67fIX2HPZKbh4ZhzuFY+Q5JKh/sOO/JhFYeCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"867e84e07b88bdee904811c3f7e80a6555911156d30722c07f3c6d3e8cb1f34c","last_reissued_at":"2026-07-05T01:34:31.974474Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:34:31.974474Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2009.05015","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:34:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iw7fpr95CP4/JAyj0Nwy6Xtr4iIe4ondzDY1VLgQNdwrsaya34Oo1foVPC6z5s4fK5ZltkdpHp2Il9Ml8jecBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T07:16:02.315584Z"},"content_sha256":"51c95170ff183aac04ddd885be1d080d9ee4f6b61741043151831845e7c86c96","schema_version":"1.0","event_id":"sha256:51c95170ff183aac04ddd885be1d080d9ee4f6b61741043151831845e7c86c96"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:QZ7IJYD3RC665ECICHB7P2AKMV","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Bias Variance Tradeoff in Analysis of Online Controlled Experiments","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.SE"],"primary_cat":"stat.AP","authors_text":"Ali Mahmoudzadeh, Paul Luo Li, Sol Sadeghi, Somit Gupta, Sophia Liu","submitted_at":"2020-09-10T17:28:14Z","abstract_excerpt":"Many organizations utilize large-scale online controlled experiments (OCEs) to accelerate innovation. Having high statistical power to detect small differences between control and treatment accurately is critical, as even small changes in key metrics can be worth millions of dollars or indicate user dissatisfaction for a very large number of users. For large-scale OCE, the duration is typically short (e.g. two weeks) to expedite changes and improvements to the product. In this paper, we examine two common approaches for analyzing usage data collected from users within the time window of an exp"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2009.05015","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2009.05015/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:34:31Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QY2+MKwsBpMkVkyAnf9dIYURLPfzX4dcb2srJg24/f2ZY03UMHL+rTU97M6j0EfWja6rkGnn5PcsuWpuowziBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T07:16:02.316248Z"},"content_sha256":"7c5240e4599affe64a62eb8b76ab772c3bedc56b7dc8e4a01db6de983190f48e","schema_version":"1.0","event_id":"sha256:7c5240e4599affe64a62eb8b76ab772c3bedc56b7dc8e4a01db6de983190f48e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/QZ7IJYD3RC665ECICHB7P2AKMV/bundle.json","state_url":"https://pith.science/pith/QZ7IJYD3RC665ECICHB7P2AKMV/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/QZ7IJYD3RC665ECICHB7P2AKMV/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T07:16:02Z","links":{"resolver":"https://pith.science/pith/QZ7IJYD3RC665ECICHB7P2AKMV","bundle":"https://pith.science/pith/QZ7IJYD3RC665ECICHB7P2AKMV/bundle.json","state":"https://pith.science/pith/QZ7IJYD3RC665ECICHB7P2AKMV/state.json","well_known_bundle":"https://pith.science/.well-known/pith/QZ7IJYD3RC665ECICHB7P2AKMV/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:QZ7IJYD3RC665ECICHB7P2AKMV","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"5b11b41b633c20077215bc380656703bb17019a0cc7c64b6c2bea360e769c06a","cross_cats_sorted":["cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.AP","submitted_at":"2020-09-10T17:28:14Z","title_canon_sha256":"c3aa34dab0f947b0814e8e1a215a36e09a0588c7c729e9bbe8ae78e1b12e82ec"},"schema_version":"1.0","source":{"id":"2009.05015","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2009.05015","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"arxiv_version","alias_value":"2009.05015v1","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2009.05015","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"pith_short_12","alias_value":"QZ7IJYD3RC66","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"pith_short_16","alias_value":"QZ7IJYD3RC665ECI","created_at":"2026-07-05T01:34:31Z"},{"alias_kind":"pith_short_8","alias_value":"QZ7IJYD3","created_at":"2026-07-05T01:34:31Z"}],"graph_snapshots":[{"event_id":"sha256:7c5240e4599affe64a62eb8b76ab772c3bedc56b7dc8e4a01db6de983190f48e","target":"graph","created_at":"2026-07-05T01:34:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2009.05015/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Many organizations utilize large-scale online controlled experiments (OCEs) to accelerate innovation. Having high statistical power to detect small differences between control and treatment accurately is critical, as even small changes in key metrics can be worth millions of dollars or indicate user dissatisfaction for a very large number of users. For large-scale OCE, the duration is typically short (e.g. two weeks) to expedite changes and improvements to the product. In this paper, we examine two common approaches for analyzing usage data collected from users within the time window of an exp","authors_text":"Ali Mahmoudzadeh, Paul Luo Li, Sol Sadeghi, Somit Gupta, Sophia Liu","cross_cats":["cs.SE"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.AP","submitted_at":"2020-09-10T17:28:14Z","title":"Bias Variance Tradeoff in Analysis of Online Controlled Experiments"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2009.05015","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:51c95170ff183aac04ddd885be1d080d9ee4f6b61741043151831845e7c86c96","target":"record","created_at":"2026-07-05T01:34:31Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"5b11b41b633c20077215bc380656703bb17019a0cc7c64b6c2bea360e769c06a","cross_cats_sorted":["cs.SE"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.AP","submitted_at":"2020-09-10T17:28:14Z","title_canon_sha256":"c3aa34dab0f947b0814e8e1a215a36e09a0588c7c729e9bbe8ae78e1b12e82ec"},"schema_version":"1.0","source":{"id":"2009.05015","kind":"arxiv","version":1}},"canonical_sha256":"867e84e07b88bdee904811c3f7e80a6555911156d30722c07f3c6d3e8cb1f34c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"867e84e07b88bdee904811c3f7e80a6555911156d30722c07f3c6d3e8cb1f34c","first_computed_at":"2026-07-05T01:34:31.974474Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:34:31.974474Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"gU0onY6D1nh4tBG5W+wo1gN2tnx2G109osVqf0+QwMTyHSW2x67fIX2HPZKbh4ZhzuFY+Q5JKh/sOO/JhFYeCA==","signature_status":"signed_v1","signed_at":"2026-07-05T01:34:31.974834Z","signed_message":"canonical_sha256_bytes"},"source_id":"2009.05015","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:51c95170ff183aac04ddd885be1d080d9ee4f6b61741043151831845e7c86c96","sha256:7c5240e4599affe64a62eb8b76ab772c3bedc56b7dc8e4a01db6de983190f48e"],"state_sha256":"4c0c13724af43f72f499ad1237f96cd9dd3f004d2ce384ce4f160c5b6936aef1"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nF/rKQp/EdH0gBnIiQhaPicY72Wk1OuuXWCUp0080WfOQv6HIyYwyVC+AoIO4gYfDqdPe4Rtc+0cGlm7l/h6Dg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T07:16:02.319608Z","bundle_sha256":"c9b440b08ae710733e3dcf8760f4f0164c5891eb1f06aafbcbedad1743386621"}}