{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2017:5VS5XC7HMMFBDBKVEUJQQ7OKMR","short_pith_number":"pith:5VS5XC7H","canonical_record":{"source":{"id":"1710.08217","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2017-10-23T11:50:11Z","cross_cats_sorted":[],"title_canon_sha256":"68e083cc7e768f27c06eef2ca699ca7637e88254af5bfc0871d7a022ae21e5b3","abstract_canon_sha256":"c5af4ddfb3a9e032696f3a6a39f22ee33efacf2073a5f011d8909472bf0fec95"},"schema_version":"1.0"},"canonical_sha256":"ed65db8be7630a1185552513087dca64611223f46c6a4319741977b56af0ec95","source":{"kind":"arxiv","id":"1710.08217","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1710.08217","created_at":"2026-05-18T00:32:16Z"},{"alias_kind":"arxiv_version","alias_value":"1710.08217v1","created_at":"2026-05-18T00:32:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1710.08217","created_at":"2026-05-18T00:32:16Z"},{"alias_kind":"pith_short_12","alias_value":"5VS5XC7HMMFB","created_at":"2026-05-18T12:31:00Z"},{"alias_kind":"pith_short_16","alias_value":"5VS5XC7HMMFBDBKV","created_at":"2026-05-18T12:31:00Z"},{"alias_kind":"pith_short_8","alias_value":"5VS5XC7H","created_at":"2026-05-18T12:31:00Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2017:5VS5XC7HMMFBDBKVEUJQQ7OKMR","target":"record","payload":{"canonical_record":{"source":{"id":"1710.08217","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2017-10-23T11:50:11Z","cross_cats_sorted":[],"title_canon_sha256":"68e083cc7e768f27c06eef2ca699ca7637e88254af5bfc0871d7a022ae21e5b3","abstract_canon_sha256":"c5af4ddfb3a9e032696f3a6a39f22ee33efacf2073a5f011d8909472bf0fec95"},"schema_version":"1.0"},"canonical_sha256":"ed65db8be7630a1185552513087dca64611223f46c6a4319741977b56af0ec95","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:32:16.933136Z","signature_b64":"CbDpFVY2vBLIcp/PSJ3NJRY14H2nSGVjclHHBUzcDIMu8mqCtvRqgAia08nIidlWHwlmOTtwwIvKuKE4yd8XDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"ed65db8be7630a1185552513087dca64611223f46c6a4319741977b56af0ec95","last_reissued_at":"2026-05-18T00:32:16.932670Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:32:16.932670Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1710.08217","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:32:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GChQ0sFKHaU/FMbL7ngDwiCYgJSYpF3OYa9oeGkLmDNMxGChsodesssoGyulbgtMs4X51tTUdRUYhzUHnvQLBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T23:48:49.780156Z"},"content_sha256":"96eb377cd74859c11e034cecd5c0df855ed2b2ffacb0d96495d7083e2ca4cf25","schema_version":"1.0","event_id":"sha256:96eb377cd74859c11e034cecd5c0df855ed2b2ffacb0d96495d7083e2ca4cf25"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2017:5VS5XC7HMMFBDBKVEUJQQ7OKMR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Democratizing online controlled experiments at Booking.com","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.HC","authors_text":"Jegar Pitchforth, Lukas Vermeer, Raphael Lopez Kaufman","submitted_at":"2017-10-23T11:50:11Z","abstract_excerpt":"There is an extensive literature about online controlled experiments, both on the statistical methods available to analyze experiment results as well as on the infrastructure built by several large scale Internet companies but also on the organizational challenges of embracing online experiments to inform product development. At Booking.com we have been conducting evidenced based product development using online experiments for more than ten years. Our methods and infrastructure were designed from their inception to reflect Booking.com culture, that is, with democratization and decentralizatio"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1710.08217","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:32:16Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"zmQBq4I4cWYvmkZ/sOtDneoBqB+yxVbT6e0TzkwVfv7KdfE59+SymKqV/obBCrQLkWkBeqGMJUupIfWlo3zjCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T23:48:49.780866Z"},"content_sha256":"26f80c8d330fd80416910761425a2ced9088df1dca7b9d39c914e06b69f99ee2","schema_version":"1.0","event_id":"sha256:26f80c8d330fd80416910761425a2ced9088df1dca7b9d39c914e06b69f99ee2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/5VS5XC7HMMFBDBKVEUJQQ7OKMR/bundle.json","state_url":"https://pith.science/pith/5VS5XC7HMMFBDBKVEUJQQ7OKMR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/5VS5XC7HMMFBDBKVEUJQQ7OKMR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T23:48:49Z","links":{"resolver":"https://pith.science/pith/5VS5XC7HMMFBDBKVEUJQQ7OKMR","bundle":"https://pith.science/pith/5VS5XC7HMMFBDBKVEUJQQ7OKMR/bundle.json","state":"https://pith.science/pith/5VS5XC7HMMFBDBKVEUJQQ7OKMR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/5VS5XC7HMMFBDBKVEUJQQ7OKMR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2017:5VS5XC7HMMFBDBKVEUJQQ7OKMR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"c5af4ddfb3a9e032696f3a6a39f22ee33efacf2073a5f011d8909472bf0fec95","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2017-10-23T11:50:11Z","title_canon_sha256":"68e083cc7e768f27c06eef2ca699ca7637e88254af5bfc0871d7a022ae21e5b3"},"schema_version":"1.0","source":{"id":"1710.08217","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1710.08217","created_at":"2026-05-18T00:32:16Z"},{"alias_kind":"arxiv_version","alias_value":"1710.08217v1","created_at":"2026-05-18T00:32:16Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1710.08217","created_at":"2026-05-18T00:32:16Z"},{"alias_kind":"pith_short_12","alias_value":"5VS5XC7HMMFB","created_at":"2026-05-18T12:31:00Z"},{"alias_kind":"pith_short_16","alias_value":"5VS5XC7HMMFBDBKV","created_at":"2026-05-18T12:31:00Z"},{"alias_kind":"pith_short_8","alias_value":"5VS5XC7H","created_at":"2026-05-18T12:31:00Z"}],"graph_snapshots":[{"event_id":"sha256:26f80c8d330fd80416910761425a2ced9088df1dca7b9d39c914e06b69f99ee2","target":"graph","created_at":"2026-05-18T00:32:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"There is an extensive literature about online controlled experiments, both on the statistical methods available to analyze experiment results as well as on the infrastructure built by several large scale Internet companies but also on the organizational challenges of embracing online experiments to inform product development. At Booking.com we have been conducting evidenced based product development using online experiments for more than ten years. Our methods and infrastructure were designed from their inception to reflect Booking.com culture, that is, with democratization and decentralizatio","authors_text":"Jegar Pitchforth, Lukas Vermeer, Raphael Lopez Kaufman","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2017-10-23T11:50:11Z","title":"Democratizing online controlled experiments at Booking.com"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1710.08217","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:96eb377cd74859c11e034cecd5c0df855ed2b2ffacb0d96495d7083e2ca4cf25","target":"record","created_at":"2026-05-18T00:32:16Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"c5af4ddfb3a9e032696f3a6a39f22ee33efacf2073a5f011d8909472bf0fec95","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2017-10-23T11:50:11Z","title_canon_sha256":"68e083cc7e768f27c06eef2ca699ca7637e88254af5bfc0871d7a022ae21e5b3"},"schema_version":"1.0","source":{"id":"1710.08217","kind":"arxiv","version":1}},"canonical_sha256":"ed65db8be7630a1185552513087dca64611223f46c6a4319741977b56af0ec95","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"ed65db8be7630a1185552513087dca64611223f46c6a4319741977b56af0ec95","first_computed_at":"2026-05-18T00:32:16.932670Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:32:16.932670Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"CbDpFVY2vBLIcp/PSJ3NJRY14H2nSGVjclHHBUzcDIMu8mqCtvRqgAia08nIidlWHwlmOTtwwIvKuKE4yd8XDQ==","signature_status":"signed_v1","signed_at":"2026-05-18T00:32:16.933136Z","signed_message":"canonical_sha256_bytes"},"source_id":"1710.08217","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:96eb377cd74859c11e034cecd5c0df855ed2b2ffacb0d96495d7083e2ca4cf25","sha256:26f80c8d330fd80416910761425a2ced9088df1dca7b9d39c914e06b69f99ee2"],"state_sha256":"f11128d4a6c4e0348731c3b57be4463e1d082f404e238c2a4e69286ebccdc31b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OBGg6IRTy8AtXM+Bd71C+Cj5WgVqve6BRFjtCHSVabmrFiiogXE6+pz2zxwflrbOz6rfpxKngsS2yNWWe31pCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T23:48:49.785622Z","bundle_sha256":"bbb5d96ed8670bbe7df34082c7762c02cd9ade3ee6870f8f347945914cd25f0f"}}