{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:3SMX3ST62SRWMZ6BVSJN6ZAS5Z","short_pith_number":"pith:3SMX3ST6","canonical_record":{"source":{"id":"2309.17012","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-29T06:53:10Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"a620ddcc5fa85143441ccb17663e8c56943ba1b0489f201bca618916b1b21cfb","abstract_canon_sha256":"a034799f8ecb228ca9409220e0ab867fb849a14b2e0cf5a70294dc1064824e9b"},"schema_version":"1.0"},"canonical_sha256":"dc997dca7ed4a36667c1ac92df6412ee61a665eb4b6b8060eef74f828f8cb206","source":{"kind":"arxiv","id":"2309.17012","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.17012","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"arxiv_version","alias_value":"2309.17012v3","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.17012","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"pith_short_12","alias_value":"3SMX3ST62SRW","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"pith_short_16","alias_value":"3SMX3ST62SRWMZ6B","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"pith_short_8","alias_value":"3SMX3ST6","created_at":"2026-07-05T09:11:22Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:3SMX3ST62SRWMZ6BVSJN6ZAS5Z","target":"record","payload":{"canonical_record":{"source":{"id":"2309.17012","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-29T06:53:10Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"a620ddcc5fa85143441ccb17663e8c56943ba1b0489f201bca618916b1b21cfb","abstract_canon_sha256":"a034799f8ecb228ca9409220e0ab867fb849a14b2e0cf5a70294dc1064824e9b"},"schema_version":"1.0"},"canonical_sha256":"dc997dca7ed4a36667c1ac92df6412ee61a665eb4b6b8060eef74f828f8cb206","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:11:22.269496Z","signature_b64":"XWHglaydpPh6EWpJ7yiMzGj0Telewv71SwHaTLAP75p7p5KaoCqjKT3brSMdTYBxAkS7KlhZT7lUILnQ6a1PCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"dc997dca7ed4a36667c1ac92df6412ee61a665eb4b6b8060eef74f828f8cb206","last_reissued_at":"2026-07-05T09:11:22.269055Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:11:22.269055Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.17012","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:11:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4rBf91uTHtvVngnMDkPKiChTnLEXIMcr3UFfOQ92gdYUeDGdaADrpg77xV08DQoDmhCTJL2Qsb5AF1kh9JqjDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T23:52:06.830122Z"},"content_sha256":"ba7e557ead11a106aca41da2659f4c63ff7505ab5d21bd8f6cfb285783495afa","schema_version":"1.0","event_id":"sha256:ba7e557ead11a106aca41da2659f4c63ff7505ab5d21bd8f6cfb285783495afa"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:3SMX3ST62SRWMZ6BVSJN6ZAS5Z","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Benchmarking Cognitive Biases in Large Language Models as Evaluators","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Dongyeop Kang, Jong Inn Park, Minhwa Lee, Ryan Koo, Vipul Raheja, Zae Myung Kim","submitted_at":"2023-09-29T06:53:10Z","abstract_excerpt":"Large Language Models are cognitively biased judges. Large Language Models (LLMs) have recently been shown to be effective as automatic evaluators with simple prompting and in-context learning. In this work, we assemble 15 LLMs of four different size ranges and evaluate their output responses by preference ranking from the other LLMs as evaluators, such as System Star is better than System Square. We then evaluate the quality of ranking outputs introducing the Cognitive Bias Benchmark for LLMs as Evaluators (CoBBLEr), a benchmark to measure six different cognitive biases in LLM evaluation outp"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.17012","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.17012/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:11:22Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"80VCaYi5MPEK+kJ+r/+/OE3IthgKDf5dKQKSHmNBUxl6svlw1qRYZKsxmbsH9zJj2oSmvNwle/f6tOvWVDaVBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T23:52:06.830606Z"},"content_sha256":"57b6be6943194e542ce58316d306f350919a4392e0f123d4d4acb62114a914cc","schema_version":"1.0","event_id":"sha256:57b6be6943194e542ce58316d306f350919a4392e0f123d4d4acb62114a914cc"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/3SMX3ST62SRWMZ6BVSJN6ZAS5Z/bundle.json","state_url":"https://pith.science/pith/3SMX3ST62SRWMZ6BVSJN6ZAS5Z/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/3SMX3ST62SRWMZ6BVSJN6ZAS5Z/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T23:52:06Z","links":{"resolver":"https://pith.science/pith/3SMX3ST62SRWMZ6BVSJN6ZAS5Z","bundle":"https://pith.science/pith/3SMX3ST62SRWMZ6BVSJN6ZAS5Z/bundle.json","state":"https://pith.science/pith/3SMX3ST62SRWMZ6BVSJN6ZAS5Z/state.json","well_known_bundle":"https://pith.science/.well-known/pith/3SMX3ST62SRWMZ6BVSJN6ZAS5Z/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:3SMX3ST62SRWMZ6BVSJN6ZAS5Z","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a034799f8ecb228ca9409220e0ab867fb849a14b2e0cf5a70294dc1064824e9b","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-29T06:53:10Z","title_canon_sha256":"a620ddcc5fa85143441ccb17663e8c56943ba1b0489f201bca618916b1b21cfb"},"schema_version":"1.0","source":{"id":"2309.17012","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.17012","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"arxiv_version","alias_value":"2309.17012v3","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.17012","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"pith_short_12","alias_value":"3SMX3ST62SRW","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"pith_short_16","alias_value":"3SMX3ST62SRWMZ6B","created_at":"2026-07-05T09:11:22Z"},{"alias_kind":"pith_short_8","alias_value":"3SMX3ST6","created_at":"2026-07-05T09:11:22Z"}],"graph_snapshots":[{"event_id":"sha256:57b6be6943194e542ce58316d306f350919a4392e0f123d4d4acb62114a914cc","target":"graph","created_at":"2026-07-05T09:11:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.17012/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models are cognitively biased judges. Large Language Models (LLMs) have recently been shown to be effective as automatic evaluators with simple prompting and in-context learning. In this work, we assemble 15 LLMs of four different size ranges and evaluate their output responses by preference ranking from the other LLMs as evaluators, such as System Star is better than System Square. We then evaluate the quality of ranking outputs introducing the Cognitive Bias Benchmark for LLMs as Evaluators (CoBBLEr), a benchmark to measure six different cognitive biases in LLM evaluation outp","authors_text":"Dongyeop Kang, Jong Inn Park, Minhwa Lee, Ryan Koo, Vipul Raheja, Zae Myung Kim","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-29T06:53:10Z","title":"Benchmarking Cognitive Biases in Large Language Models as Evaluators"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.17012","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:ba7e557ead11a106aca41da2659f4c63ff7505ab5d21bd8f6cfb285783495afa","target":"record","created_at":"2026-07-05T09:11:22Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a034799f8ecb228ca9409220e0ab867fb849a14b2e0cf5a70294dc1064824e9b","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-29T06:53:10Z","title_canon_sha256":"a620ddcc5fa85143441ccb17663e8c56943ba1b0489f201bca618916b1b21cfb"},"schema_version":"1.0","source":{"id":"2309.17012","kind":"arxiv","version":3}},"canonical_sha256":"dc997dca7ed4a36667c1ac92df6412ee61a665eb4b6b8060eef74f828f8cb206","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"dc997dca7ed4a36667c1ac92df6412ee61a665eb4b6b8060eef74f828f8cb206","first_computed_at":"2026-07-05T09:11:22.269055Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:11:22.269055Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"XWHglaydpPh6EWpJ7yiMzGj0Telewv71SwHaTLAP75p7p5KaoCqjKT3brSMdTYBxAkS7KlhZT7lUILnQ6a1PCA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:11:22.269496Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.17012","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:ba7e557ead11a106aca41da2659f4c63ff7505ab5d21bd8f6cfb285783495afa","sha256:57b6be6943194e542ce58316d306f350919a4392e0f123d4d4acb62114a914cc"],"state_sha256":"8c6350685d4d8b7e679fbe88f72403c128983bd1c6bc6d5212ec0b62edf96727"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Lrg73drtVOuC6buuRYOYIg0fP5oRbnU+PY8NbL7u8GiTxzkqFc/YVARtiWZQMvSQqeciIuH+FuzQ1igcpEAoBA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T23:52:06.833803Z","bundle_sha256":"864f9a290905b9f759a54c4d24380f78f36bebb5692f6456ba752d640ea5cfa1"}}