{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2020:NROLVR6WJCSBIIDSDBCSGKVPRF","short_pith_number":"pith:NROLVR6W","canonical_record":{"source":{"id":"2011.02804","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2020-11-05T13:20:45Z","cross_cats_sorted":[],"title_canon_sha256":"7adc3e0a834837e05166a1fd993c4e5159530d403e140f0435adc94a8bbccebf","abstract_canon_sha256":"14274ca35f371c7e40ab13ee19df61d168c549d232b3df11f027883731ffe61e"},"schema_version":"1.0"},"canonical_sha256":"6c5cbac7d648a41420721845232aaf894fae0ffb58c18f2867ca0c7ae3ed2c06","source":{"kind":"arxiv","id":"2011.02804","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2011.02804","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"arxiv_version","alias_value":"2011.02804v1","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2011.02804","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"pith_short_12","alias_value":"NROLVR6WJCSB","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"pith_short_16","alias_value":"NROLVR6WJCSBIIDS","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"pith_short_8","alias_value":"NROLVR6W","created_at":"2026-07-05T01:49:25Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2020:NROLVR6WJCSBIIDSDBCSGKVPRF","target":"record","payload":{"canonical_record":{"source":{"id":"2011.02804","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2020-11-05T13:20:45Z","cross_cats_sorted":[],"title_canon_sha256":"7adc3e0a834837e05166a1fd993c4e5159530d403e140f0435adc94a8bbccebf","abstract_canon_sha256":"14274ca35f371c7e40ab13ee19df61d168c549d232b3df11f027883731ffe61e"},"schema_version":"1.0"},"canonical_sha256":"6c5cbac7d648a41420721845232aaf894fae0ffb58c18f2867ca0c7ae3ed2c06","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T01:49:25.926856Z","signature_b64":"vr4/TLf8MQjcq/tJnQbhcO60oxCAT/OTRkGxE7D+Jn1iZyExcKlm8nTqEvAacBpOnZq4LMf+qmkk+onL/6PCBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"6c5cbac7d648a41420721845232aaf894fae0ffb58c18f2867ca0c7ae3ed2c06","last_reissued_at":"2026-07-05T01:49:25.926442Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T01:49:25.926442Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2011.02804","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:49:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"XThPST+43nNFFROoil9zhfzZZSxWovtmBriUM0bEQAvdzriaIbC2enMR3Ieh3Ul+607zl5hVtq8C54pmjTSCBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T02:05:54.026254Z"},"content_sha256":"adc318e3d62f71ce70d173cc2dbdd9e38bb7de571d75736175847abdc4366725","schema_version":"1.0","event_id":"sha256:adc318e3d62f71ce70d173cc2dbdd9e38bb7de571d75736175847abdc4366725"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2020:NROLVR6WJCSBIIDSDBCSGKVPRF","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Challenges and strategies for running controlled crowdsourcing experiments","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.HC","authors_text":"Boualem Benatallah, Fabio Casati, Jorge Ram\\'irez, Luca Cernuzzi, Marcos Baez","submitted_at":"2020-11-05T13:20:45Z","abstract_excerpt":"This paper reports on the challenges and lessons we learned while running controlled experiments in crowdsourcing platforms. Crowdsourcing is becoming an attractive technique to engage a diverse and large pool of subjects in experimental research, allowing researchers to achieve levels of scale and completion times that would otherwise not be feasible in lab settings. However, the scale and flexibility comes at the cost of multiple and sometimes unknown sources of bias and confounding factors that arise from technical limitations of crowdsourcing platforms and from the challenges of running co"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2011.02804","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2011.02804/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T01:49:25Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"kUKuhCu9/vWkw1L0BX4rOUDf+UlGn2YB1aH8mUzwLh5Bxt2RERgZDctmYpryrNwN+EKTrEbhxo25m+fJC5x3AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-09T02:05:54.026680Z"},"content_sha256":"4d60ef6fa5c245a9025ea45ad9bc6f607a5b3e61bbb338df611070ee3260acd4","schema_version":"1.0","event_id":"sha256:4d60ef6fa5c245a9025ea45ad9bc6f607a5b3e61bbb338df611070ee3260acd4"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NROLVR6WJCSBIIDSDBCSGKVPRF/bundle.json","state_url":"https://pith.science/pith/NROLVR6WJCSBIIDSDBCSGKVPRF/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NROLVR6WJCSBIIDSDBCSGKVPRF/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-09T02:05:54Z","links":{"resolver":"https://pith.science/pith/NROLVR6WJCSBIIDSDBCSGKVPRF","bundle":"https://pith.science/pith/NROLVR6WJCSBIIDSDBCSGKVPRF/bundle.json","state":"https://pith.science/pith/NROLVR6WJCSBIIDSDBCSGKVPRF/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NROLVR6WJCSBIIDSDBCSGKVPRF/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2020:NROLVR6WJCSBIIDSDBCSGKVPRF","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"14274ca35f371c7e40ab13ee19df61d168c549d232b3df11f027883731ffe61e","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2020-11-05T13:20:45Z","title_canon_sha256":"7adc3e0a834837e05166a1fd993c4e5159530d403e140f0435adc94a8bbccebf"},"schema_version":"1.0","source":{"id":"2011.02804","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2011.02804","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"arxiv_version","alias_value":"2011.02804v1","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2011.02804","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"pith_short_12","alias_value":"NROLVR6WJCSB","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"pith_short_16","alias_value":"NROLVR6WJCSBIIDS","created_at":"2026-07-05T01:49:25Z"},{"alias_kind":"pith_short_8","alias_value":"NROLVR6W","created_at":"2026-07-05T01:49:25Z"}],"graph_snapshots":[{"event_id":"sha256:4d60ef6fa5c245a9025ea45ad9bc6f607a5b3e61bbb338df611070ee3260acd4","target":"graph","created_at":"2026-07-05T01:49:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2011.02804/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"This paper reports on the challenges and lessons we learned while running controlled experiments in crowdsourcing platforms. Crowdsourcing is becoming an attractive technique to engage a diverse and large pool of subjects in experimental research, allowing researchers to achieve levels of scale and completion times that would otherwise not be feasible in lab settings. However, the scale and flexibility comes at the cost of multiple and sometimes unknown sources of bias and confounding factors that arise from technical limitations of crowdsourcing platforms and from the challenges of running co","authors_text":"Boualem Benatallah, Fabio Casati, Jorge Ram\\'irez, Luca Cernuzzi, Marcos Baez","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2020-11-05T13:20:45Z","title":"Challenges and strategies for running controlled crowdsourcing experiments"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2011.02804","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:adc318e3d62f71ce70d173cc2dbdd9e38bb7de571d75736175847abdc4366725","target":"record","created_at":"2026-07-05T01:49:25Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"14274ca35f371c7e40ab13ee19df61d168c549d232b3df11f027883731ffe61e","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2020-11-05T13:20:45Z","title_canon_sha256":"7adc3e0a834837e05166a1fd993c4e5159530d403e140f0435adc94a8bbccebf"},"schema_version":"1.0","source":{"id":"2011.02804","kind":"arxiv","version":1}},"canonical_sha256":"6c5cbac7d648a41420721845232aaf894fae0ffb58c18f2867ca0c7ae3ed2c06","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"6c5cbac7d648a41420721845232aaf894fae0ffb58c18f2867ca0c7ae3ed2c06","first_computed_at":"2026-07-05T01:49:25.926442Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T01:49:25.926442Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"vr4/TLf8MQjcq/tJnQbhcO60oxCAT/OTRkGxE7D+Jn1iZyExcKlm8nTqEvAacBpOnZq4LMf+qmkk+onL/6PCBA==","signature_status":"signed_v1","signed_at":"2026-07-05T01:49:25.926856Z","signed_message":"canonical_sha256_bytes"},"source_id":"2011.02804","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:adc318e3d62f71ce70d173cc2dbdd9e38bb7de571d75736175847abdc4366725","sha256:4d60ef6fa5c245a9025ea45ad9bc6f607a5b3e61bbb338df611070ee3260acd4"],"state_sha256":"a069dd2fec678baf034edf01eb85b0a9680f5579be2ca56d9886fd1b5252fc7f"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Cd9a2wQxNOHwAf9Ovh/wkbVCYPJfMTWMhcK0BFl+zea6+jithSxLviSuTxYKTs5GHjrlM+LlQ7zPoU+PQ6/ZDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-09T02:05:54.028906Z","bundle_sha256":"cc0c95a51c56d1f7f95f5715a4ed2daa779e66557c6baf645439a045a6c9e54f"}}