{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:VSWSIS5G2RZKD7QP2DUR6VLQSM","short_pith_number":"pith:VSWSIS5G","canonical_record":{"source":{"id":"2403.16950","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-25T17:11:28Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"626292c20883a5a3427ff6d0c38731951076738aae74fac6bca944828a66ad58","abstract_canon_sha256":"beebf2153b2871d95f97c4b4ea64478ef73b526559130df4e7f42fc0904403d4"},"schema_version":"1.0"},"canonical_sha256":"acad244ba6d472a1fe0fd0e91f55709311c6aec69f0e8fb5af15a92dce96082a","source":{"kind":"arxiv","id":"2403.16950","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.16950","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"arxiv_version","alias_value":"2403.16950v5","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.16950","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"pith_short_12","alias_value":"VSWSIS5G2RZK","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"pith_short_16","alias_value":"VSWSIS5G2RZKD7QP","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"pith_short_8","alias_value":"VSWSIS5G","created_at":"2026-07-05T10:02:02Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:VSWSIS5G2RZKD7QP2DUR6VLQSM","target":"record","payload":{"canonical_record":{"source":{"id":"2403.16950","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-25T17:11:28Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"626292c20883a5a3427ff6d0c38731951076738aae74fac6bca944828a66ad58","abstract_canon_sha256":"beebf2153b2871d95f97c4b4ea64478ef73b526559130df4e7f42fc0904403d4"},"schema_version":"1.0"},"canonical_sha256":"acad244ba6d472a1fe0fd0e91f55709311c6aec69f0e8fb5af15a92dce96082a","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:02:02.834431Z","signature_b64":"tZw/3H6RcKYm5aUbcW2o5yOz4EDXZgMHHkwbobft/urMz8Y7jUqNrUdbwxp5kPUvnirySCyPxswBdZdatcaICw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"acad244ba6d472a1fe0fd0e91f55709311c6aec69f0e8fb5af15a92dce96082a","last_reissued_at":"2026-07-05T10:02:02.833997Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:02:02.833997Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2403.16950","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:02:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Gk81T9Kp7/TlSrKhXiP7p+Vv2J4I6w/S0vc9tUAOGBL3rxil+BwgA538GYP3ScLRoslXMHyPauAnvjiR23j8Aw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T04:29:11.981320Z"},"content_sha256":"e22899dc6f534182a0568f5283325701c0627d8a0c661076c38f6de19bc24893","schema_version":"1.0","event_id":"sha256:e22899dc6f534182a0568f5283325701c0627d8a0c661076c38f6de19bc24893"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:VSWSIS5G2RZKD7QP2DUR6VLQSM","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Aligning with Human Judgement: The Role of Pairwise Preference in Large Language Model Evaluators","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Anna Korhonen, Ehsan Shareghi, Han Zhou, Ivan Vuli\\'c, Nigel Collier, Yinhong Liu, Zhijiang Guo","submitted_at":"2024-03-25T17:11:28Z","abstract_excerpt":"Large Language Models (LLMs) have demonstrated promising capabilities as automatic evaluators in assessing the quality of generated natural language. However, LLMs still exhibit biases in evaluation and often struggle to generate coherent evaluations that align with human assessments. In this work, we first conduct a systematic study of the misalignment between LLM evaluators and human evaluation, revealing that existing calibration methods aimed at mitigating biases of LLMs are insufficient for effectively aligning LLM evaluators. Inspired by the use of preference data in RLHF, we formulate t"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.16950","kind":"arxiv","version":5},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2403.16950/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:02:02Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D2CzZQ2e7w/O4FjC4cDH3+iNs4mBrmUZnoOuWpswcXN/UvmiOVSmApJnpmS7v+BJJvgO9vjeUuPNOdFaJoFVCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-06T04:29:11.981745Z"},"content_sha256":"7bbd80b1e7b6f0b614fd4703edce72250ce0f48078684140481d147468f6ae9e","schema_version":"1.0","event_id":"sha256:7bbd80b1e7b6f0b614fd4703edce72250ce0f48078684140481d147468f6ae9e"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VSWSIS5G2RZKD7QP2DUR6VLQSM/bundle.json","state_url":"https://pith.science/pith/VSWSIS5G2RZKD7QP2DUR6VLQSM/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VSWSIS5G2RZKD7QP2DUR6VLQSM/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-06T04:29:11Z","links":{"resolver":"https://pith.science/pith/VSWSIS5G2RZKD7QP2DUR6VLQSM","bundle":"https://pith.science/pith/VSWSIS5G2RZKD7QP2DUR6VLQSM/bundle.json","state":"https://pith.science/pith/VSWSIS5G2RZKD7QP2DUR6VLQSM/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VSWSIS5G2RZKD7QP2DUR6VLQSM/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:VSWSIS5G2RZKD7QP2DUR6VLQSM","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"beebf2153b2871d95f97c4b4ea64478ef73b526559130df4e7f42fc0904403d4","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-25T17:11:28Z","title_canon_sha256":"626292c20883a5a3427ff6d0c38731951076738aae74fac6bca944828a66ad58"},"schema_version":"1.0","source":{"id":"2403.16950","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2403.16950","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"arxiv_version","alias_value":"2403.16950v5","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2403.16950","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"pith_short_12","alias_value":"VSWSIS5G2RZK","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"pith_short_16","alias_value":"VSWSIS5G2RZKD7QP","created_at":"2026-07-05T10:02:02Z"},{"alias_kind":"pith_short_8","alias_value":"VSWSIS5G","created_at":"2026-07-05T10:02:02Z"}],"graph_snapshots":[{"event_id":"sha256:7bbd80b1e7b6f0b614fd4703edce72250ce0f48078684140481d147468f6ae9e","target":"graph","created_at":"2026-07-05T10:02:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2403.16950/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have demonstrated promising capabilities as automatic evaluators in assessing the quality of generated natural language. However, LLMs still exhibit biases in evaluation and often struggle to generate coherent evaluations that align with human assessments. In this work, we first conduct a systematic study of the misalignment between LLM evaluators and human evaluation, revealing that existing calibration methods aimed at mitigating biases of LLMs are insufficient for effectively aligning LLM evaluators. Inspired by the use of preference data in RLHF, we formulate t","authors_text":"Anna Korhonen, Ehsan Shareghi, Han Zhou, Ivan Vuli\\'c, Nigel Collier, Yinhong Liu, Zhijiang Guo","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-25T17:11:28Z","title":"Aligning with Human Judgement: The Role of Pairwise Preference in Large Language Model Evaluators"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2403.16950","kind":"arxiv","version":5},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:e22899dc6f534182a0568f5283325701c0627d8a0c661076c38f6de19bc24893","target":"record","created_at":"2026-07-05T10:02:02Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"beebf2153b2871d95f97c4b4ea64478ef73b526559130df4e7f42fc0904403d4","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-03-25T17:11:28Z","title_canon_sha256":"626292c20883a5a3427ff6d0c38731951076738aae74fac6bca944828a66ad58"},"schema_version":"1.0","source":{"id":"2403.16950","kind":"arxiv","version":5}},"canonical_sha256":"acad244ba6d472a1fe0fd0e91f55709311c6aec69f0e8fb5af15a92dce96082a","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"acad244ba6d472a1fe0fd0e91f55709311c6aec69f0e8fb5af15a92dce96082a","first_computed_at":"2026-07-05T10:02:02.833997Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:02:02.833997Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"tZw/3H6RcKYm5aUbcW2o5yOz4EDXZgMHHkwbobft/urMz8Y7jUqNrUdbwxp5kPUvnirySCyPxswBdZdatcaICw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:02:02.834431Z","signed_message":"canonical_sha256_bytes"},"source_id":"2403.16950","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:e22899dc6f534182a0568f5283325701c0627d8a0c661076c38f6de19bc24893","sha256:7bbd80b1e7b6f0b614fd4703edce72250ce0f48078684140481d147468f6ae9e"],"state_sha256":"2436c9dbf59bd91e66b11545937dc916fbd4ca2516a9269c711a6c5a70635678"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VQfrrLVhh3FTBmrn9AO2AIwguyOzBLlzGszl/jIJxBQJP9ooY7QZVq3b7ZK2bPMR9u+vB0APgmotyvfX5WJbBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-06T04:29:11.985621Z","bundle_sha256":"bfbda0febce5523883515963cd7d63952f01170c5d7e941f241ede88b31353ec"}}