{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:FF6EUINSLDTXYHKVZHZCRNC5QR","short_pith_number":"pith:FF6EUINS","canonical_record":{"source":{"id":"2109.06835","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2021-09-14T17:20:30Z","cross_cats_sorted":[],"title_canon_sha256":"647218d44f13c1dc00edbb75454c5bf349786f11fccffec5badc42e2b87ce415","abstract_canon_sha256":"51b2bf46caba9c1ba2ac594d6e122ad9522f317f6a8415664592d7213f483cb1"},"schema_version":"1.0"},"canonical_sha256":"297c4a21b258e77c1d55c9f228b45d84548ecc9e787f39f6237c64688196a8a8","source":{"kind":"arxiv","id":"2109.06835","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2109.06835","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"arxiv_version","alias_value":"2109.06835v1","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.06835","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"pith_short_12","alias_value":"FF6EUINSLDTX","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"pith_short_16","alias_value":"FF6EUINSLDTXYHKV","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"pith_short_8","alias_value":"FF6EUINS","created_at":"2026-07-05T03:14:30Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:FF6EUINSLDTXYHKVZHZCRNC5QR","target":"record","payload":{"canonical_record":{"source":{"id":"2109.06835","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2021-09-14T17:20:30Z","cross_cats_sorted":[],"title_canon_sha256":"647218d44f13c1dc00edbb75454c5bf349786f11fccffec5badc42e2b87ce415","abstract_canon_sha256":"51b2bf46caba9c1ba2ac594d6e122ad9522f317f6a8415664592d7213f483cb1"},"schema_version":"1.0"},"canonical_sha256":"297c4a21b258e77c1d55c9f228b45d84548ecc9e787f39f6237c64688196a8a8","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T03:14:30.690712Z","signature_b64":"6jJ8Vn6tOxSQEu7eAQmfba9PfoerKKvKB1rvH+FA11z6YfKF1eNBnPk1nD+2GISU9zPe9jW6rI53Oe3SMJtIDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"297c4a21b258e77c1d55c9f228b45d84548ecc9e787f39f6237c64688196a8a8","last_reissued_at":"2026-07-05T03:14:30.690287Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T03:14:30.690287Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2109.06835","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:14:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"FP4AUG5oQRtWZVbM9MZZbNj0tzaNvIMhElNfEfnoTTuGveIWVxQ2TrJ+wPlrGDN6oL1Tl/DgZG9NNSQEcK8hBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T01:47:24.602180Z"},"content_sha256":"0afe8320bbc9bd0c18573018c36a88555d6d55451cb83e0390ca5e5b2aae2544","schema_version":"1.0","event_id":"sha256:0afe8320bbc9bd0c18573018c36a88555d6d55451cb83e0390ca5e5b2aae2544"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:FF6EUINSLDTXYHKVZHZCRNC5QR","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"The Perils of Using Mechanical Turk to Evaluate Open-Ended Text Generation","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Marzena Karpinska, Mohit Iyyer, Nader Akoury","submitted_at":"2021-09-14T17:20:30Z","abstract_excerpt":"Recent text generation research has increasingly focused on open-ended domains such as story and poetry generation. Because models built for such tasks are difficult to evaluate automatically, most researchers in the space justify their modeling choices by collecting crowdsourced human judgments of text quality (e.g., Likert scores of coherence or grammaticality) from Amazon Mechanical Turk (AMT). In this paper, we first conduct a survey of 45 open-ended text generation papers and find that the vast majority of them fail to report crucial details about their AMT tasks, hindering reproducibilit"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2109.06835","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2109.06835/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T03:14:30Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"pLj5rUHobp1Ofd8E5Ne2I9Kiapl+ebMyo8MlNQy1Xmt+lIjE2Uj9GJKIt5n3vdEiH8d5FsiVbbXxKi+rcDC6Ag==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-14T01:47:24.603111Z"},"content_sha256":"e35540cce993ea1378fa1d42d1dcb5c0b7ee56a652e544a65de111b48ba844cb","schema_version":"1.0","event_id":"sha256:e35540cce993ea1378fa1d42d1dcb5c0b7ee56a652e544a65de111b48ba844cb"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/FF6EUINSLDTXYHKVZHZCRNC5QR/bundle.json","state_url":"https://pith.science/pith/FF6EUINSLDTXYHKVZHZCRNC5QR/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/FF6EUINSLDTXYHKVZHZCRNC5QR/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-14T01:47:24Z","links":{"resolver":"https://pith.science/pith/FF6EUINSLDTXYHKVZHZCRNC5QR","bundle":"https://pith.science/pith/FF6EUINSLDTXYHKVZHZCRNC5QR/bundle.json","state":"https://pith.science/pith/FF6EUINSLDTXYHKVZHZCRNC5QR/state.json","well_known_bundle":"https://pith.science/.well-known/pith/FF6EUINSLDTXYHKVZHZCRNC5QR/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:FF6EUINSLDTXYHKVZHZCRNC5QR","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"51b2bf46caba9c1ba2ac594d6e122ad9522f317f6a8415664592d7213f483cb1","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2021-09-14T17:20:30Z","title_canon_sha256":"647218d44f13c1dc00edbb75454c5bf349786f11fccffec5badc42e2b87ce415"},"schema_version":"1.0","source":{"id":"2109.06835","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2109.06835","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"arxiv_version","alias_value":"2109.06835v1","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2109.06835","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"pith_short_12","alias_value":"FF6EUINSLDTX","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"pith_short_16","alias_value":"FF6EUINSLDTXYHKV","created_at":"2026-07-05T03:14:30Z"},{"alias_kind":"pith_short_8","alias_value":"FF6EUINS","created_at":"2026-07-05T03:14:30Z"}],"graph_snapshots":[{"event_id":"sha256:e35540cce993ea1378fa1d42d1dcb5c0b7ee56a652e544a65de111b48ba844cb","target":"graph","created_at":"2026-07-05T03:14:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2109.06835/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Recent text generation research has increasingly focused on open-ended domains such as story and poetry generation. Because models built for such tasks are difficult to evaluate automatically, most researchers in the space justify their modeling choices by collecting crowdsourced human judgments of text quality (e.g., Likert scores of coherence or grammaticality) from Amazon Mechanical Turk (AMT). In this paper, we first conduct a survey of 45 open-ended text generation papers and find that the vast majority of them fail to report crucial details about their AMT tasks, hindering reproducibilit","authors_text":"Marzena Karpinska, Mohit Iyyer, Nader Akoury","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2021-09-14T17:20:30Z","title":"The Perils of Using Mechanical Turk to Evaluate Open-Ended Text Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2109.06835","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:0afe8320bbc9bd0c18573018c36a88555d6d55451cb83e0390ca5e5b2aae2544","target":"record","created_at":"2026-07-05T03:14:30Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"51b2bf46caba9c1ba2ac594d6e122ad9522f317f6a8415664592d7213f483cb1","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-nd/4.0/","primary_cat":"cs.CL","submitted_at":"2021-09-14T17:20:30Z","title_canon_sha256":"647218d44f13c1dc00edbb75454c5bf349786f11fccffec5badc42e2b87ce415"},"schema_version":"1.0","source":{"id":"2109.06835","kind":"arxiv","version":1}},"canonical_sha256":"297c4a21b258e77c1d55c9f228b45d84548ecc9e787f39f6237c64688196a8a8","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"297c4a21b258e77c1d55c9f228b45d84548ecc9e787f39f6237c64688196a8a8","first_computed_at":"2026-07-05T03:14:30.690287Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T03:14:30.690287Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"6jJ8Vn6tOxSQEu7eAQmfba9PfoerKKvKB1rvH+FA11z6YfKF1eNBnPk1nD+2GISU9zPe9jW6rI53Oe3SMJtIDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T03:14:30.690712Z","signed_message":"canonical_sha256_bytes"},"source_id":"2109.06835","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:0afe8320bbc9bd0c18573018c36a88555d6d55451cb83e0390ca5e5b2aae2544","sha256:e35540cce993ea1378fa1d42d1dcb5c0b7ee56a652e544a65de111b48ba844cb"],"state_sha256":"9a8383c8d3615f6a570ac9b0953c13d3f6fbd9db6d1c35e1821bbc632e513f69"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"OcgHTQq5MxYXNkQb8t8KPk+KRcY5jnK09J9kTON2T1SQwYoUmir6aNOxZB9O6AK1NISK3VH291p5tHiZf3e7CA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-14T01:47:24.610416Z","bundle_sha256":"8aadf2f64ba5da40bb4d1ca03784c5f283c5b19ee1497a21cc32eb93722a99f4"}}