{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2021:LIGLZA36TBD7WEHQKW3A2WF6AW","short_pith_number":"pith:LIGLZA36","canonical_record":{"source":{"id":"2101.06561","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-01-17T00:40:47Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a1ce6c238491f571b144a3accdc0d6a00baf2cc5e9aa074e13c1baa781c4ef0a","abstract_canon_sha256":"143a98940bf0db579cbfdcdb2357bf89233631295d9f6a4a1c631fabea5c2eb5"},"schema_version":"1.0"},"canonical_sha256":"5a0cbc837e9847fb10f055b60d58be05957de5ea84b169b444a48d06e2cf826f","source":{"kind":"arxiv","id":"2101.06561","version":4},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2101.06561","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"arxiv_version","alias_value":"2101.06561v4","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2101.06561","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"pith_short_12","alias_value":"LIGLZA36TBD7","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"pith_short_16","alias_value":"LIGLZA36TBD7WEHQ","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"pith_short_8","alias_value":"LIGLZA36","created_at":"2026-07-05T05:12:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2021:LIGLZA36TBD7WEHQKW3A2WF6AW","target":"record","payload":{"canonical_record":{"source":{"id":"2101.06561","kind":"arxiv","version":4},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-01-17T00:40:47Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"a1ce6c238491f571b144a3accdc0d6a00baf2cc5e9aa074e13c1baa781c4ef0a","abstract_canon_sha256":"143a98940bf0db579cbfdcdb2357bf89233631295d9f6a4a1c631fabea5c2eb5"},"schema_version":"1.0"},"canonical_sha256":"5a0cbc837e9847fb10f055b60d58be05957de5ea84b169b444a48d06e2cf826f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T05:12:07.607602Z","signature_b64":"NCAALHNdPdUKZFtaYxuwpnq2qACqqypJwx8nzT9Ind8gh2eirO6AADrtj30r+buOzLmUD7FocYURkw0S68EICg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5a0cbc837e9847fb10f055b60d58be05957de5ea84b169b444a48d06e2cf826f","last_reissued_at":"2026-07-05T05:12:07.607185Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T05:12:07.607185Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2101.06561","source_version":4,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"qrf1P26ThWWxXU+lr7ukGr9wS2TvJA3TzzC7hu4J/CI7XJrJJZiI533svFfFwhAJTeZnzu5rI+Ig1Pv5mY1bCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T16:31:51.411776Z"},"content_sha256":"e06949b0105c81f0c7b4e44a7965263d69ce382fc6debf7da159eacc2b2afc47","schema_version":"1.0","event_id":"sha256:e06949b0105c81f0c7b4e44a7965263d69ce382fc6debf7da159eacc2b2afc47"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2021:LIGLZA36TBD7WEHQKW3A2WF6AW","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"GENIE: Toward Reproducible and Standardized Human Evaluation for Text Generation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Daniel Khashabi, Daniel S. Weld, Gabriel Stanovsky, Jonathan Bragg, Jungo Kasai, Nicholas Lourie, Noah A. Smith, Yejin Choi","submitted_at":"2021-01-17T00:40:47Z","abstract_excerpt":"While often assumed a gold standard, effective human evaluation of text generation remains an important, open area for research. We revisit this problem with a focus on producing consistent evaluations that are reproducible -- over time and across different populations. We study this goal in different stages of the human evaluation pipeline. In particular, we consider design choices for the annotation interface used to elicit human judgments and their impact on reproducibility. Furthermore, we develop an automated mechanism for maintaining annotator quality via a probabilistic model that detec"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2101.06561","kind":"arxiv","version":4},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2101.06561/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T05:12:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YrYelRDnrvWWb2XvQn/12VIxj0CJ8QfvpySe/ZbOclbROGEmOrv5WHHiqL7xElOpaO5IuBTCNIiR5sOU0UUmDA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-13T16:31:51.412310Z"},"content_sha256":"2ba904e14a00a49ec470ba7df63e56a7c40ce38bfece4c610c486ffe4251be43","schema_version":"1.0","event_id":"sha256:2ba904e14a00a49ec470ba7df63e56a7c40ce38bfece4c610c486ffe4251be43"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/LIGLZA36TBD7WEHQKW3A2WF6AW/bundle.json","state_url":"https://pith.science/pith/LIGLZA36TBD7WEHQKW3A2WF6AW/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/LIGLZA36TBD7WEHQKW3A2WF6AW/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-13T16:31:51Z","links":{"resolver":"https://pith.science/pith/LIGLZA36TBD7WEHQKW3A2WF6AW","bundle":"https://pith.science/pith/LIGLZA36TBD7WEHQKW3A2WF6AW/bundle.json","state":"https://pith.science/pith/LIGLZA36TBD7WEHQKW3A2WF6AW/state.json","well_known_bundle":"https://pith.science/.well-known/pith/LIGLZA36TBD7WEHQKW3A2WF6AW/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2021:LIGLZA36TBD7WEHQKW3A2WF6AW","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"143a98940bf0db579cbfdcdb2357bf89233631295d9f6a4a1c631fabea5c2eb5","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-01-17T00:40:47Z","title_canon_sha256":"a1ce6c238491f571b144a3accdc0d6a00baf2cc5e9aa074e13c1baa781c4ef0a"},"schema_version":"1.0","source":{"id":"2101.06561","kind":"arxiv","version":4}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2101.06561","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"arxiv_version","alias_value":"2101.06561v4","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2101.06561","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"pith_short_12","alias_value":"LIGLZA36TBD7","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"pith_short_16","alias_value":"LIGLZA36TBD7WEHQ","created_at":"2026-07-05T05:12:07Z"},{"alias_kind":"pith_short_8","alias_value":"LIGLZA36","created_at":"2026-07-05T05:12:07Z"}],"graph_snapshots":[{"event_id":"sha256:2ba904e14a00a49ec470ba7df63e56a7c40ce38bfece4c610c486ffe4251be43","target":"graph","created_at":"2026-07-05T05:12:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2101.06561/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While often assumed a gold standard, effective human evaluation of text generation remains an important, open area for research. We revisit this problem with a focus on producing consistent evaluations that are reproducible -- over time and across different populations. We study this goal in different stages of the human evaluation pipeline. In particular, we consider design choices for the annotation interface used to elicit human judgments and their impact on reproducibility. Furthermore, we develop an automated mechanism for maintaining annotator quality via a probabilistic model that detec","authors_text":"Daniel Khashabi, Daniel S. Weld, Gabriel Stanovsky, Jonathan Bragg, Jungo Kasai, Nicholas Lourie, Noah A. Smith, Yejin Choi","cross_cats":["cs.AI"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-01-17T00:40:47Z","title":"GENIE: Toward Reproducible and Standardized Human Evaluation for Text Generation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2101.06561","kind":"arxiv","version":4},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e06949b0105c81f0c7b4e44a7965263d69ce382fc6debf7da159eacc2b2afc47","target":"record","created_at":"2026-07-05T05:12:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"143a98940bf0db579cbfdcdb2357bf89233631295d9f6a4a1c631fabea5c2eb5","cross_cats_sorted":["cs.AI"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2021-01-17T00:40:47Z","title_canon_sha256":"a1ce6c238491f571b144a3accdc0d6a00baf2cc5e9aa074e13c1baa781c4ef0a"},"schema_version":"1.0","source":{"id":"2101.06561","kind":"arxiv","version":4}},"canonical_sha256":"5a0cbc837e9847fb10f055b60d58be05957de5ea84b169b444a48d06e2cf826f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5a0cbc837e9847fb10f055b60d58be05957de5ea84b169b444a48d06e2cf826f","first_computed_at":"2026-07-05T05:12:07.607185Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T05:12:07.607185Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"NCAALHNdPdUKZFtaYxuwpnq2qACqqypJwx8nzT9Ind8gh2eirO6AADrtj30r+buOzLmUD7FocYURkw0S68EICg==","signature_status":"signed_v1","signed_at":"2026-07-05T05:12:07.607602Z","signed_message":"canonical_sha256_bytes"},"source_id":"2101.06561","source_kind":"arxiv","source_version":4}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e06949b0105c81f0c7b4e44a7965263d69ce382fc6debf7da159eacc2b2afc47","sha256:2ba904e14a00a49ec470ba7df63e56a7c40ce38bfece4c610c486ffe4251be43"],"state_sha256":"2857ec52c900ba5749730abd01c71a1c85c78bef53afa06b840fcbf959d7f914"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"658E0+wZRnfLIEKJnp0Mkyt+WyvdfpMK9hZakJ7c6OT6iLLs+tbCI65iqTkK3oBrKsapKgkN9JWxZg39Bie0AA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-13T16:31:51.416921Z","bundle_sha256":"b6f65941a09ab4c0746125bb322bda3f02da8b96827321d8cba770c4b4c9b7b6"}}