{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:EJOAOPE65WC5NW3H7PUYXEA46T","short_pith_number":"pith:EJOAOPE6","canonical_record":{"source":{"id":"2402.10524","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2024-02-16T09:14:49Z","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"title_canon_sha256":"e697cfe9f92bb1b15f50eedb924fe9016612c04bcd7fd3173876696b67337110","abstract_canon_sha256":"65595bd1123275b3930ac9b24b38e09108c23137bc2f5664e1da48a18f6547c1"},"schema_version":"1.0"},"canonical_sha256":"225c073c9eed85d6db67fbe98b901cf4e8b5aac24d332cf658973e07d97c0fa2","source":{"kind":"arxiv","id":"2402.10524","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.10524","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"arxiv_version","alias_value":"2402.10524v1","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.10524","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"pith_short_12","alias_value":"EJOAOPE65WC5","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"pith_short_16","alias_value":"EJOAOPE65WC5NW3H","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"pith_short_8","alias_value":"EJOAOPE6","created_at":"2026-07-05T07:45:57Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:EJOAOPE65WC5NW3H7PUYXEA46T","target":"record","payload":{"canonical_record":{"source":{"id":"2402.10524","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2024-02-16T09:14:49Z","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"title_canon_sha256":"e697cfe9f92bb1b15f50eedb924fe9016612c04bcd7fd3173876696b67337110","abstract_canon_sha256":"65595bd1123275b3930ac9b24b38e09108c23137bc2f5664e1da48a18f6547c1"},"schema_version":"1.0"},"canonical_sha256":"225c073c9eed85d6db67fbe98b901cf4e8b5aac24d332cf658973e07d97c0fa2","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:45:57.524592Z","signature_b64":"csoKpRTOHVLAjXJzzHrdIr8SWkMsoE8pQJqMuELjPx7xt6ZrCBF91wHl9Amfc2TIvM1woh3d9nfAjO18+9y0DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"225c073c9eed85d6db67fbe98b901cf4e8b5aac24d332cf658973e07d97c0fa2","last_reissued_at":"2026-07-05T07:45:57.524189Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:45:57.524189Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2402.10524","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:45:57Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KjeogSRZahoaJ64jVQyYmj8thXzql2srJjWo8+RDpJ2HOkMvOjpc7CHCwD3pDXT+L5k/rMotm7rX5/L7rXv1Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T14:08:14.117308Z"},"content_sha256":"05c1f8079fd89902ddeb4e5ec5e0e8a5a1bdd768bdde4797967cba2523dd6601","schema_version":"1.0","event_id":"sha256:05c1f8079fd89902ddeb4e5ec5e0e8a5a1bdd768bdde4797967cba2523dd6601"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:EJOAOPE65WC5NW3H7PUYXEA46T","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"LLM Comparator: Visual Analytics for Side-by-Side Evaluation of Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.LG"],"primary_cat":"cs.HC","authors_text":"Emily Reif, Ian Tenney, James Wexler, Krystal Kallarackal, Lucas Dixon, Mahima Pushkarna, Michael Terry, Michael Xieyang Liu, Minsuk Chang, Minsuk Kahng","submitted_at":"2024-02-16T09:14:49Z","abstract_excerpt":"Automatic side-by-side evaluation has emerged as a promising approach to evaluating the quality of responses from large language models (LLMs). However, analyzing the results from this evaluation approach raises scalability and interpretability challenges. In this paper, we present LLM Comparator, a novel visual analytics tool for interactively analyzing results from automatic side-by-side evaluation. The tool supports interactive workflows for users to understand when and why a model performs better or worse than a baseline model, and how the responses from two models are qualitatively differ"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.10524","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2402.10524/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:45:57Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"tpddTEV4/1eKVaozfFCUNdwWYkXPcoSqhwP2+VyI51q8mT5CQo1ZzUQxS4eilIS6FNNfdeDtU5eqB+tt808MBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-04T14:08:14.118621Z"},"content_sha256":"343a8c3c87b0ca78bfcaefbddd27428d4eb9d38ca1b0d473600cc53c9e97053c","schema_version":"1.0","event_id":"sha256:343a8c3c87b0ca78bfcaefbddd27428d4eb9d38ca1b0d473600cc53c9e97053c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/EJOAOPE65WC5NW3H7PUYXEA46T/bundle.json","state_url":"https://pith.science/pith/EJOAOPE65WC5NW3H7PUYXEA46T/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/EJOAOPE65WC5NW3H7PUYXEA46T/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-04T14:08:14Z","links":{"resolver":"https://pith.science/pith/EJOAOPE65WC5NW3H7PUYXEA46T","bundle":"https://pith.science/pith/EJOAOPE65WC5NW3H7PUYXEA46T/bundle.json","state":"https://pith.science/pith/EJOAOPE65WC5NW3H7PUYXEA46T/state.json","well_known_bundle":"https://pith.science/.well-known/pith/EJOAOPE65WC5NW3H7PUYXEA46T/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:EJOAOPE65WC5NW3H7PUYXEA46T","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"65595bd1123275b3930ac9b24b38e09108c23137bc2f5664e1da48a18f6547c1","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2024-02-16T09:14:49Z","title_canon_sha256":"e697cfe9f92bb1b15f50eedb924fe9016612c04bcd7fd3173876696b67337110"},"schema_version":"1.0","source":{"id":"2402.10524","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2402.10524","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"arxiv_version","alias_value":"2402.10524v1","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2402.10524","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"pith_short_12","alias_value":"EJOAOPE65WC5","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"pith_short_16","alias_value":"EJOAOPE65WC5NW3H","created_at":"2026-07-05T07:45:57Z"},{"alias_kind":"pith_short_8","alias_value":"EJOAOPE6","created_at":"2026-07-05T07:45:57Z"}],"graph_snapshots":[{"event_id":"sha256:343a8c3c87b0ca78bfcaefbddd27428d4eb9d38ca1b0d473600cc53c9e97053c","target":"graph","created_at":"2026-07-05T07:45:57Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2402.10524/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Automatic side-by-side evaluation has emerged as a promising approach to evaluating the quality of responses from large language models (LLMs). However, analyzing the results from this evaluation approach raises scalability and interpretability challenges. In this paper, we present LLM Comparator, a novel visual analytics tool for interactively analyzing results from automatic side-by-side evaluation. The tool supports interactive workflows for users to understand when and why a model performs better or worse than a baseline model, and how the responses from two models are qualitatively differ","authors_text":"Emily Reif, Ian Tenney, James Wexler, Krystal Kallarackal, Lucas Dixon, Mahima Pushkarna, Michael Terry, Michael Xieyang Liu, Minsuk Chang, Minsuk Kahng","cross_cats":["cs.AI","cs.CL","cs.LG"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2024-02-16T09:14:49Z","title":"LLM Comparator: Visual Analytics for Side-by-Side Evaluation of Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2402.10524","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:05c1f8079fd89902ddeb4e5ec5e0e8a5a1bdd768bdde4797967cba2523dd6601","target":"record","created_at":"2026-07-05T07:45:57Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"65595bd1123275b3930ac9b24b38e09108c23137bc2f5664e1da48a18f6547c1","cross_cats_sorted":["cs.AI","cs.CL","cs.LG"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.HC","submitted_at":"2024-02-16T09:14:49Z","title_canon_sha256":"e697cfe9f92bb1b15f50eedb924fe9016612c04bcd7fd3173876696b67337110"},"schema_version":"1.0","source":{"id":"2402.10524","kind":"arxiv","version":1}},"canonical_sha256":"225c073c9eed85d6db67fbe98b901cf4e8b5aac24d332cf658973e07d97c0fa2","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"225c073c9eed85d6db67fbe98b901cf4e8b5aac24d332cf658973e07d97c0fa2","first_computed_at":"2026-07-05T07:45:57.524189Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:45:57.524189Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"csoKpRTOHVLAjXJzzHrdIr8SWkMsoE8pQJqMuELjPx7xt6ZrCBF91wHl9Amfc2TIvM1woh3d9nfAjO18+9y0DA==","signature_status":"signed_v1","signed_at":"2026-07-05T07:45:57.524592Z","signed_message":"canonical_sha256_bytes"},"source_id":"2402.10524","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:05c1f8079fd89902ddeb4e5ec5e0e8a5a1bdd768bdde4797967cba2523dd6601","sha256:343a8c3c87b0ca78bfcaefbddd27428d4eb9d38ca1b0d473600cc53c9e97053c"],"state_sha256":"021060132e787d03ae688bbee1cc6af8ad2defbff583b8ade794446c74a431a5"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Z73jdqC7zXACXHhGC1xrwfEP2cJU5jm+g55aGaLAWpRH3ELHiLn84fsim2A5qet602Cu5ozrKuf3ejvxDqHJAg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-04T14:08:14.141234Z","bundle_sha256":"38534400e6ac9a708ef87eecff47372930154a0e9bc8d3b3b7bfdd430864ebbf"}}