{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:GY5JIL7WA6NASW7WU6CLQYQ4UD","short_pith_number":"pith:GY5JIL7W","canonical_record":{"source":{"id":"2405.18638","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T22:45:28Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"7311cf396fd359edcf4d731e080a4eec7791151cd181b4760138df0e33107301","abstract_canon_sha256":"f6c7d580b055a0111b4cd2da33a0d2736741c7209a296dbee3762df5b95cca90"},"schema_version":"1.0"},"canonical_sha256":"363a942ff6079a095bf6a784b8621ca0eb82984d27909a392ff97a7542feb10b","source":{"kind":"arxiv","id":"2405.18638","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.18638","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"arxiv_version","alias_value":"2405.18638v2","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.18638","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"pith_short_12","alias_value":"GY5JIL7WA6NA","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"pith_short_16","alias_value":"GY5JIL7WA6NASW7W","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"pith_short_8","alias_value":"GY5JIL7W","created_at":"2026-07-05T09:16:15Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:GY5JIL7WA6NASW7WU6CLQYQ4UD","target":"record","payload":{"canonical_record":{"source":{"id":"2405.18638","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T22:45:28Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"7311cf396fd359edcf4d731e080a4eec7791151cd181b4760138df0e33107301","abstract_canon_sha256":"f6c7d580b055a0111b4cd2da33a0d2736741c7209a296dbee3762df5b95cca90"},"schema_version":"1.0"},"canonical_sha256":"363a942ff6079a095bf6a784b8621ca0eb82984d27909a392ff97a7542feb10b","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:16:15.569032Z","signature_b64":"zWt1FQnZ3aXi78P/0NSFfM0Of8Df4KK84Zo9YsBPojn4wv+REzH904tacIH0h7h7urHl1TLghgMFiwu4oVxBAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"363a942ff6079a095bf6a784b8621ca0eb82984d27909a392ff97a7542feb10b","last_reissued_at":"2026-07-05T09:16:15.568368Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:16:15.568368Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2405.18638","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:16:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"CicNoTTRFWWALD0TNNMmTv6+g2xXoNQJm9Ee6RFkB3KA2Qea4VGP47x2vS7hTLirU3Gk8rpcUsBwx9M7lLMVBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T17:47:53.679388Z"},"content_sha256":"d9e19ef9bc46a3295da11eca30328428d172c750c06611c7661dc64d60e7d5c0","schema_version":"1.0","event_id":"sha256:d9e19ef9bc46a3295da11eca30328428d172c750c06611c7661dc64d60e7d5c0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:GY5JIL7WA6NASW7WU6CLQYQ4UD","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"ConSiDERS-The-Human Evaluation Framework: Rethinking Human Evaluation for Generative Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Aparna Elangovan, Dan Roth, Lei Xu, Ling Liu, Sravan Bodapati","submitted_at":"2024-05-28T22:45:28Z","abstract_excerpt":"In this position paper, we argue that human evaluation of generative large language models (LLMs) should be a multidisciplinary undertaking that draws upon insights from disciplines such as user experience research and human behavioral psychology to ensure that the experimental design and results are reliable. The conclusions from these evaluations, thus, must consider factors such as usability, aesthetics, and cognitive biases. We highlight how cognitive biases can conflate fluent information and truthfulness, and how cognitive uncertainty affects the reliability of rating scores such as Like"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.18638","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2405.18638/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:16:15Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NlYF0zJKFdy4HjEhJM35BIzosC5htEACKrpcuv7PgtcsL2N1mtgnrselg2ebsUevP5n5wcnPCu574WD7sBSSDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T17:47:53.679774Z"},"content_sha256":"bf77078f799d6c7cb493311718c11092fc4cc3c1bf84184c98da6e8ec6365a38","schema_version":"1.0","event_id":"sha256:bf77078f799d6c7cb493311718c11092fc4cc3c1bf84184c98da6e8ec6365a38"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GY5JIL7WA6NASW7WU6CLQYQ4UD/bundle.json","state_url":"https://pith.science/pith/GY5JIL7WA6NASW7WU6CLQYQ4UD/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GY5JIL7WA6NASW7WU6CLQYQ4UD/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T17:47:53Z","links":{"resolver":"https://pith.science/pith/GY5JIL7WA6NASW7WU6CLQYQ4UD","bundle":"https://pith.science/pith/GY5JIL7WA6NASW7WU6CLQYQ4UD/bundle.json","state":"https://pith.science/pith/GY5JIL7WA6NASW7WU6CLQYQ4UD/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GY5JIL7WA6NASW7WU6CLQYQ4UD/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GY5JIL7WA6NASW7WU6CLQYQ4UD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"f6c7d580b055a0111b4cd2da33a0d2736741c7209a296dbee3762df5b95cca90","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T22:45:28Z","title_canon_sha256":"7311cf396fd359edcf4d731e080a4eec7791151cd181b4760138df0e33107301"},"schema_version":"1.0","source":{"id":"2405.18638","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2405.18638","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"arxiv_version","alias_value":"2405.18638v2","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2405.18638","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"pith_short_12","alias_value":"GY5JIL7WA6NA","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"pith_short_16","alias_value":"GY5JIL7WA6NASW7W","created_at":"2026-07-05T09:16:15Z"},{"alias_kind":"pith_short_8","alias_value":"GY5JIL7W","created_at":"2026-07-05T09:16:15Z"}],"graph_snapshots":[{"event_id":"sha256:bf77078f799d6c7cb493311718c11092fc4cc3c1bf84184c98da6e8ec6365a38","target":"graph","created_at":"2026-07-05T09:16:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2405.18638/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this position paper, we argue that human evaluation of generative large language models (LLMs) should be a multidisciplinary undertaking that draws upon insights from disciplines such as user experience research and human behavioral psychology to ensure that the experimental design and results are reliable. The conclusions from these evaluations, thus, must consider factors such as usability, aesthetics, and cognitive biases. We highlight how cognitive biases can conflate fluent information and truthfulness, and how cognitive uncertainty affects the reliability of rating scores such as Like","authors_text":"Aparna Elangovan, Dan Roth, Lei Xu, Ling Liu, Sravan Bodapati","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T22:45:28Z","title":"ConSiDERS-The-Human Evaluation Framework: Rethinking Human Evaluation for Generative Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2405.18638","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d9e19ef9bc46a3295da11eca30328428d172c750c06611c7661dc64d60e7d5c0","target":"record","created_at":"2026-07-05T09:16:15Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"f6c7d580b055a0111b4cd2da33a0d2736741c7209a296dbee3762df5b95cca90","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-05-28T22:45:28Z","title_canon_sha256":"7311cf396fd359edcf4d731e080a4eec7791151cd181b4760138df0e33107301"},"schema_version":"1.0","source":{"id":"2405.18638","kind":"arxiv","version":2}},"canonical_sha256":"363a942ff6079a095bf6a784b8621ca0eb82984d27909a392ff97a7542feb10b","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"363a942ff6079a095bf6a784b8621ca0eb82984d27909a392ff97a7542feb10b","first_computed_at":"2026-07-05T09:16:15.568368Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:16:15.568368Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"zWt1FQnZ3aXi78P/0NSFfM0Of8Df4KK84Zo9YsBPojn4wv+REzH904tacIH0h7h7urHl1TLghgMFiwu4oVxBAg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:16:15.569032Z","signed_message":"canonical_sha256_bytes"},"source_id":"2405.18638","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d9e19ef9bc46a3295da11eca30328428d172c750c06611c7661dc64d60e7d5c0","sha256:bf77078f799d6c7cb493311718c11092fc4cc3c1bf84184c98da6e8ec6365a38"],"state_sha256":"6db328bca577f5491315f098b255b2e31493f5a08a42144a8c0ab44f907c722d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"hf0iUHMSRoXahW/CxBnYTksPvR/hffwHgAc9pnsTomJARxwV9+UOCZJck3JhdildwLFxtEehLHC8T1FI2UKoCg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T17:47:53.682737Z","bundle_sha256":"bcaecea21e67680311c495fc651ed540988bcbb0da8655599685f705ba71d1b1"}}