{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:Q4R7THSTY2VGDADP2QO4E3WTY7","short_pith_number":"pith:Q4R7THST","canonical_record":{"source":{"id":"2407.03841","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T11:14:47Z","cross_cats_sorted":[],"title_canon_sha256":"41a9c8082ac725d2d5cbc67942a9f3375fea29d33000de2f77b8047ce70668c2","abstract_canon_sha256":"2ca5b185ce9c60e18d2aadc0016aa547f9d9c4e81eca0573ecc7a42b5b738fb7"},"schema_version":"1.0"},"canonical_sha256":"8723f99e53c6aa61806fd41dc26ed3c7cac3c53db7c229e1c7b63acf856a8420","source":{"kind":"arxiv","id":"2407.03841","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.03841","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"arxiv_version","alias_value":"2407.03841v1","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.03841","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"pith_short_12","alias_value":"Q4R7THSTY2VG","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"pith_short_16","alias_value":"Q4R7THSTY2VGDADP","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"pith_short_8","alias_value":"Q4R7THST","created_at":"2026-07-05T08:40:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:Q4R7THSTY2VGDADP2QO4E3WTY7","target":"record","payload":{"canonical_record":{"source":{"id":"2407.03841","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T11:14:47Z","cross_cats_sorted":[],"title_canon_sha256":"41a9c8082ac725d2d5cbc67942a9f3375fea29d33000de2f77b8047ce70668c2","abstract_canon_sha256":"2ca5b185ce9c60e18d2aadc0016aa547f9d9c4e81eca0573ecc7a42b5b738fb7"},"schema_version":"1.0"},"canonical_sha256":"8723f99e53c6aa61806fd41dc26ed3c7cac3c53db7c229e1c7b63acf856a8420","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:40:24.820851Z","signature_b64":"dOwfpCpi9GnRE1JjvZzBlAQB3+NGlOORAMKj08elVdHd+cCE4JenGyQFNrFWOTe4Z2gmXxP8XLohVyneIJMrBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8723f99e53c6aa61806fd41dc26ed3c7cac3c53db7c229e1c7b63acf856a8420","last_reissued_at":"2026-07-05T08:40:24.820377Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:40:24.820377Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.03841","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:40:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"3Tyh6XKn5l9Jtt//fX85jdru5r4tvVewcc5Bhhq/rHXP6R9CcDli2yk1+reTw34iABW+Nqu/Xri/6NcwEDxHDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T23:10:27.125623Z"},"content_sha256":"66b8de308ac78a55f279a24f083341bd089b1fde56acf34db012a0da875a6c88","schema_version":"1.0","event_id":"sha256:66b8de308ac78a55f279a24f083341bd089b1fde56acf34db012a0da875a6c88"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:Q4R7THSTY2VGDADP2QO4E3WTY7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"On the Benchmarking of LLMs for Open-Domain Dialogue Evaluation","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alon Lavie, Isabel Trancoso, John Mendon\\c{c}a","submitted_at":"2024-07-04T11:14:47Z","abstract_excerpt":"Large Language Models (LLMs) have showcased remarkable capabilities in various Natural Language Processing tasks. For automatic open-domain dialogue evaluation in particular, LLMs have been seamlessly integrated into evaluation frameworks, and together with human evaluation, compose the backbone of most evaluations. However, existing evaluation benchmarks often rely on outdated datasets and evaluate aspects like Fluency and Relevance, which fail to adequately capture the capabilities and limitations of state-of-the-art chatbot models.\n  This paper critically examines current evaluation benchma"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.03841","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.03841/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:40:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Xdujv/W5SSQaGwoZLtppx5tzQSynbwohqUHcSlpPCd4zMZstDaFGZ0ZhCYk37DbyCLIBj9FsfCj5rAcOgWV+CA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-06T23:10:27.126008Z"},"content_sha256":"31fd9ae9a7f444797027be255462f40f7f1dd79c756ed4a9430708b3b93cd157","schema_version":"1.0","event_id":"sha256:31fd9ae9a7f444797027be255462f40f7f1dd79c756ed4a9430708b3b93cd157"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/Q4R7THSTY2VGDADP2QO4E3WTY7/bundle.json","state_url":"https://pith.science/pith/Q4R7THSTY2VGDADP2QO4E3WTY7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/Q4R7THSTY2VGDADP2QO4E3WTY7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-06T23:10:27Z","links":{"resolver":"https://pith.science/pith/Q4R7THSTY2VGDADP2QO4E3WTY7","bundle":"https://pith.science/pith/Q4R7THSTY2VGDADP2QO4E3WTY7/bundle.json","state":"https://pith.science/pith/Q4R7THSTY2VGDADP2QO4E3WTY7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/Q4R7THSTY2VGDADP2QO4E3WTY7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:Q4R7THSTY2VGDADP2QO4E3WTY7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2ca5b185ce9c60e18d2aadc0016aa547f9d9c4e81eca0573ecc7a42b5b738fb7","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T11:14:47Z","title_canon_sha256":"41a9c8082ac725d2d5cbc67942a9f3375fea29d33000de2f77b8047ce70668c2"},"schema_version":"1.0","source":{"id":"2407.03841","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.03841","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"arxiv_version","alias_value":"2407.03841v1","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.03841","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"pith_short_12","alias_value":"Q4R7THSTY2VG","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"pith_short_16","alias_value":"Q4R7THSTY2VGDADP","created_at":"2026-07-05T08:40:24Z"},{"alias_kind":"pith_short_8","alias_value":"Q4R7THST","created_at":"2026-07-05T08:40:24Z"}],"graph_snapshots":[{"event_id":"sha256:31fd9ae9a7f444797027be255462f40f7f1dd79c756ed4a9430708b3b93cd157","target":"graph","created_at":"2026-07-05T08:40:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.03841/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) have showcased remarkable capabilities in various Natural Language Processing tasks. For automatic open-domain dialogue evaluation in particular, LLMs have been seamlessly integrated into evaluation frameworks, and together with human evaluation, compose the backbone of most evaluations. However, existing evaluation benchmarks often rely on outdated datasets and evaluate aspects like Fluency and Relevance, which fail to adequately capture the capabilities and limitations of state-of-the-art chatbot models.\n  This paper critically examines current evaluation benchma","authors_text":"Alon Lavie, Isabel Trancoso, John Mendon\\c{c}a","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T11:14:47Z","title":"On the Benchmarking of LLMs for Open-Domain Dialogue Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.03841","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:66b8de308ac78a55f279a24f083341bd089b1fde56acf34db012a0da875a6c88","target":"record","created_at":"2026-07-05T08:40:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2ca5b185ce9c60e18d2aadc0016aa547f9d9c4e81eca0573ecc7a42b5b738fb7","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-07-04T11:14:47Z","title_canon_sha256":"41a9c8082ac725d2d5cbc67942a9f3375fea29d33000de2f77b8047ce70668c2"},"schema_version":"1.0","source":{"id":"2407.03841","kind":"arxiv","version":1}},"canonical_sha256":"8723f99e53c6aa61806fd41dc26ed3c7cac3c53db7c229e1c7b63acf856a8420","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"8723f99e53c6aa61806fd41dc26ed3c7cac3c53db7c229e1c7b63acf856a8420","first_computed_at":"2026-07-05T08:40:24.820377Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:40:24.820377Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"dOwfpCpi9GnRE1JjvZzBlAQB3+NGlOORAMKj08elVdHd+cCE4JenGyQFNrFWOTe4Z2gmXxP8XLohVyneIJMrBw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:40:24.820851Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.03841","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:66b8de308ac78a55f279a24f083341bd089b1fde56acf34db012a0da875a6c88","sha256:31fd9ae9a7f444797027be255462f40f7f1dd79c756ed4a9430708b3b93cd157"],"state_sha256":"982d59fa3573cce59b9a93a688aed513eb764e72a7e141f62dfa3c59c9a300f0"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"WM/IlGT89mgnpW2qHvCr0PyUTUy/HekzsIEAQjlzQqKE4ovCApDHQwa0ieEuRFAiT6wqv5a2eDLc+y0LA7SSDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-06T23:10:27.127998Z","bundle_sha256":"97ed39155c98b0734731cbf7b148204d197306594c23e81813c796e60e92fd51"}}