{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2018:NQA7PKK3HG2XAQNMM67TK36N2C","short_pith_number":"pith:NQA7PKK3","canonical_record":{"source":{"id":"1808.06152","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ME","submitted_at":"2018-08-19T02:16:40Z","cross_cats_sorted":["cs.LG","stat.ML"],"title_canon_sha256":"00fb5e3415dc585fcc81090b7c3be2757fa1aa0cdadb516ba0be608769f6456e","abstract_canon_sha256":"ccca08fc4dc6050e375af970c2af9375fb8b1cdd63235b475c92ac4db3b11b40"},"schema_version":"1.0"},"canonical_sha256":"6c01f7a95b39b57041ac67bf356fcdd0b13394ebbc3ded34be0563962d6cfc3a","source":{"kind":"arxiv","id":"1808.06152","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1808.06152","created_at":"2026-05-18T00:05:04Z"},{"alias_kind":"arxiv_version","alias_value":"1808.06152v1","created_at":"2026-05-18T00:05:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1808.06152","created_at":"2026-05-18T00:05:04Z"},{"alias_kind":"pith_short_12","alias_value":"NQA7PKK3HG2X","created_at":"2026-05-18T12:32:40Z"},{"alias_kind":"pith_short_16","alias_value":"NQA7PKK3HG2XAQNM","created_at":"2026-05-18T12:32:40Z"},{"alias_kind":"pith_short_8","alias_value":"NQA7PKK3","created_at":"2026-05-18T12:32:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2018:NQA7PKK3HG2XAQNMM67TK36N2C","target":"record","payload":{"canonical_record":{"source":{"id":"1808.06152","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ME","submitted_at":"2018-08-19T02:16:40Z","cross_cats_sorted":["cs.LG","stat.ML"],"title_canon_sha256":"00fb5e3415dc585fcc81090b7c3be2757fa1aa0cdadb516ba0be608769f6456e","abstract_canon_sha256":"ccca08fc4dc6050e375af970c2af9375fb8b1cdd63235b475c92ac4db3b11b40"},"schema_version":"1.0"},"canonical_sha256":"6c01f7a95b39b57041ac67bf356fcdd0b13394ebbc3ded34be0563962d6cfc3a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-18T00:05:04.526507Z","signature_b64":"oXYY2TYtPyppaZnvmLEnXB9e7yppVivvX7utdo1vq1tTzJNjIZng2Kdb00kO/dmIWSFWqT8nhpPM+ACKc72FAw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6c01f7a95b39b57041ac67bf356fcdd0b13394ebbc3ded34be0563962d6cfc3a","last_reissued_at":"2026-05-18T00:05:04.525960Z","signature_status":"signed_v1","first_computed_at":"2026-05-18T00:05:04.525960Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1808.06152","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:05:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"dBPRbD/lHvmgG3qU1ah+XMWmMv5t/pw2tftK5Vut7iDtDKvLP2aqL02k9z9Gg/YgJbhIggtudCH2Xa26m2xDBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-23T15:20:23.844396Z"},"content_sha256":"557f26bd877ab495ea139e947b3470180d5d9c0d9f3db972d80ee1cafb6e3983","schema_version":"1.0","event_id":"sha256:557f26bd877ab495ea139e947b3470180d5d9c0d9f3db972d80ee1cafb6e3983"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2018:NQA7PKK3HG2XAQNMM67TK36N2C","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On Design of Problem Token Questions in Quality of Experience Surveys","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.LG","stat.ML"],"primary_cat":"stat.ME","authors_text":"Ebrahim Beyrami, Jayant Gupchup, Martin Ellis, Ross Cutler, Sam Johnson, Yasaman Hosseinkashi","submitted_at":"2018-08-19T02:16:40Z","abstract_excerpt":"User surveys for Quality of Experience (QoE) are a critical source of information. In addition to the common \"star rating\" used to estimate Mean Opinion Score (MOS), more detailed survey questions (problem tokens) about specific areas provide valuable insight into the factors impacting QoE. This paper explores two aspects of the problem token questionnaire design. First, we study the bias introduced by fixed question order, and second, we study the challenge of selecting a subset of questions to keep the token set small. Based on 900,000 calls gathered using a randomized controlled experiment "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1808.06152","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-18T00:05:04Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jWIa7QjDwQvIrupEqGxNmEtVQfc8YUBkRlzhMgG+CIKYKRt0AinuFbjJ7rb6E08ul6qphoqOhNcw9OEAkGG5CA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-05-23T15:20:23.845178Z"},"content_sha256":"e3143097ed947904267f162e3966012a2f857e5830d39942970170778cbbb41d","schema_version":"1.0","event_id":"sha256:e3143097ed947904267f162e3966012a2f857e5830d39942970170778cbbb41d"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NQA7PKK3HG2XAQNMM67TK36N2C/bundle.json","state_url":"https://pith.science/pith/NQA7PKK3HG2XAQNMM67TK36N2C/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NQA7PKK3HG2XAQNMM67TK36N2C/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-05-23T15:20:23Z","links":{"resolver":"https://pith.science/pith/NQA7PKK3HG2XAQNMM67TK36N2C","bundle":"https://pith.science/pith/NQA7PKK3HG2XAQNMM67TK36N2C/bundle.json","state":"https://pith.science/pith/NQA7PKK3HG2XAQNMM67TK36N2C/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NQA7PKK3HG2XAQNMM67TK36N2C/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2018:NQA7PKK3HG2XAQNMM67TK36N2C","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ccca08fc4dc6050e375af970c2af9375fb8b1cdd63235b475c92ac4db3b11b40","cross_cats_sorted":["cs.LG","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ME","submitted_at":"2018-08-19T02:16:40Z","title_canon_sha256":"00fb5e3415dc585fcc81090b7c3be2757fa1aa0cdadb516ba0be608769f6456e"},"schema_version":"1.0","source":{"id":"1808.06152","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1808.06152","created_at":"2026-05-18T00:05:04Z"},{"alias_kind":"arxiv_version","alias_value":"1808.06152v1","created_at":"2026-05-18T00:05:04Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1808.06152","created_at":"2026-05-18T00:05:04Z"},{"alias_kind":"pith_short_12","alias_value":"NQA7PKK3HG2X","created_at":"2026-05-18T12:32:40Z"},{"alias_kind":"pith_short_16","alias_value":"NQA7PKK3HG2XAQNM","created_at":"2026-05-18T12:32:40Z"},{"alias_kind":"pith_short_8","alias_value":"NQA7PKK3","created_at":"2026-05-18T12:32:40Z"}],"graph_snapshots":[{"event_id":"sha256:e3143097ed947904267f162e3966012a2f857e5830d39942970170778cbbb41d","target":"graph","created_at":"2026-05-18T00:05:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"User surveys for Quality of Experience (QoE) are a critical source of information. In addition to the common \"star rating\" used to estimate Mean Opinion Score (MOS), more detailed survey questions (problem tokens) about specific areas provide valuable insight into the factors impacting QoE. This paper explores two aspects of the problem token questionnaire design. First, we study the bias introduced by fixed question order, and second, we study the challenge of selecting a subset of questions to keep the token set small. Based on 900,000 calls gathered using a randomized controlled experiment ","authors_text":"Ebrahim Beyrami, Jayant Gupchup, Martin Ellis, Ross Cutler, Sam Johnson, Yasaman Hosseinkashi","cross_cats":["cs.LG","stat.ML"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ME","submitted_at":"2018-08-19T02:16:40Z","title":"On Design of Problem Token Questions in Quality of Experience Surveys"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1808.06152","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:557f26bd877ab495ea139e947b3470180d5d9c0d9f3db972d80ee1cafb6e3983","target":"record","created_at":"2026-05-18T00:05:04Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ccca08fc4dc6050e375af970c2af9375fb8b1cdd63235b475c92ac4db3b11b40","cross_cats_sorted":["cs.LG","stat.ML"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ME","submitted_at":"2018-08-19T02:16:40Z","title_canon_sha256":"00fb5e3415dc585fcc81090b7c3be2757fa1aa0cdadb516ba0be608769f6456e"},"schema_version":"1.0","source":{"id":"1808.06152","kind":"arxiv","version":1}},"canonical_sha256":"6c01f7a95b39b57041ac67bf356fcdd0b13394ebbc3ded34be0563962d6cfc3a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6c01f7a95b39b57041ac67bf356fcdd0b13394ebbc3ded34be0563962d6cfc3a","first_computed_at":"2026-05-18T00:05:04.525960Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:05:04.525960Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"oXYY2TYtPyppaZnvmLEnXB9e7yppVivvX7utdo1vq1tTzJNjIZng2Kdb00kO/dmIWSFWqT8nhpPM+ACKc72FAw==","signature_status":"signed_v1","signed_at":"2026-05-18T00:05:04.526507Z","signed_message":"canonical_sha256_bytes"},"source_id":"1808.06152","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:557f26bd877ab495ea139e947b3470180d5d9c0d9f3db972d80ee1cafb6e3983","sha256:e3143097ed947904267f162e3966012a2f857e5830d39942970170778cbbb41d"],"state_sha256":"962610ce11651495bccc7ec8108bf3fc0b029a2576e60e892a6e5477f4449fbe"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"F6NqiGJ2otjKEq8hE/u5wwg5YZ7muWO/L+JZUcE/Wl3k3T25pNFpp+7z2cEAMYrL1i6KrM8g/43pYmEY7P1QAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-05-23T15:20:23.849001Z","bundle_sha256":"83e04c7c39fc608701e73e1dbcf595cd149b321989d77e2d4be50dc969e64adc"}}