{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:A5VPEZSIS3DY4X567LBCWAYS77","short_pith_number":"pith:A5VPEZSI","canonical_record":{"source":{"id":"2308.04624","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-08T23:30:20Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5cf4ced1670542d1a1c6beb9f44ca8063d66f9082d13acc1a9d06ebca7cbd1bd","abstract_canon_sha256":"a56e2653462936670ce01f4230830d485989d90c274eff3495362a8f56346c8d"},"schema_version":"1.0"},"canonical_sha256":"076af2664896c78e5fbefac22b0312fffe447758df4e6e3f8fc407ff32ec18fd","source":{"kind":"arxiv","id":"2308.04624","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.04624","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"arxiv_version","alias_value":"2308.04624v1","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.04624","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"pith_short_12","alias_value":"A5VPEZSIS3DY","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"pith_short_16","alias_value":"A5VPEZSIS3DY4X56","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"pith_short_8","alias_value":"A5VPEZSI","created_at":"2026-07-05T06:39:40Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:A5VPEZSIS3DY4X567LBCWAYS77","target":"record","payload":{"canonical_record":{"source":{"id":"2308.04624","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-08T23:30:20Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"5cf4ced1670542d1a1c6beb9f44ca8063d66f9082d13acc1a9d06ebca7cbd1bd","abstract_canon_sha256":"a56e2653462936670ce01f4230830d485989d90c274eff3495362a8f56346c8d"},"schema_version":"1.0"},"canonical_sha256":"076af2664896c78e5fbefac22b0312fffe447758df4e6e3f8fc407ff32ec18fd","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:39:40.472844Z","signature_b64":"nQn/7ZF0tsiexVqQcs51n0YQN12Wg5F9W3mhxVbOlY7BMyWR+8DGbvqy1qrDKfeMvVN2wWPlaNoPJiE9U2JSAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"076af2664896c78e5fbefac22b0312fffe447758df4e6e3f8fc407ff32ec18fd","last_reissued_at":"2026-07-05T06:39:40.472374Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:39:40.472374Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2308.04624","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:39:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"D3zOOGUVwGh2zSfwkehrggiJc430N3GqOOsmgAqbF/TE7vegKB0LCZ/zU/pIyrRc51gKZ4HTGq0NYfDCagmVDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T07:25:44.552660Z"},"content_sha256":"f1df8dbc6bf97a4f53158dd544cd4d00f019bf02aaa58284ef39dfc188e74d56","schema_version":"1.0","event_id":"sha256:f1df8dbc6bf97a4f53158dd544cd4d00f019bf02aaa58284ef39dfc188e74d56"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:A5VPEZSIS3DY4X567LBCWAYS77","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Benchmarking LLM powered Chatbots: Methods and Metrics","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Arjun Avadhanam, Debarag Banerjee, Pooja Singh, Saksham Srivastava","submitted_at":"2023-08-08T23:30:20Z","abstract_excerpt":"Autonomous conversational agents, i.e. chatbots, are becoming an increasingly common mechanism for enterprises to provide support to customers and partners. In order to rate chatbots, especially ones powered by Generative AI tools like Large Language Models (LLMs) we need to be able to accurately assess their performance. This is where chatbot benchmarking becomes important. In this paper, we propose the use of a novel benchmark that we call the E2E (End to End) benchmark, and show how the E2E benchmark can be used to evaluate accuracy and usefulness of the answers provided by chatbots, especi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.04624","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2308.04624/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T06:39:40Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"sB6393NtemR4yoyRBfnHYB7S67slfBqA2Wo4mObrxhzgNOp5RAiLTYHiVX+mIN47/j6RZAN+BkpEDt5Q7SAtBQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T07:25:44.553187Z"},"content_sha256":"124c0802355557c630260a122815940246e903efab29b31f5bb50c94e699fa12","schema_version":"1.0","event_id":"sha256:124c0802355557c630260a122815940246e903efab29b31f5bb50c94e699fa12"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/A5VPEZSIS3DY4X567LBCWAYS77/bundle.json","state_url":"https://pith.science/pith/A5VPEZSIS3DY4X567LBCWAYS77/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/A5VPEZSIS3DY4X567LBCWAYS77/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T07:25:44Z","links":{"resolver":"https://pith.science/pith/A5VPEZSIS3DY4X567LBCWAYS77","bundle":"https://pith.science/pith/A5VPEZSIS3DY4X567LBCWAYS77/bundle.json","state":"https://pith.science/pith/A5VPEZSIS3DY4X567LBCWAYS77/state.json","well_known_bundle":"https://pith.science/.well-known/pith/A5VPEZSIS3DY4X567LBCWAYS77/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:A5VPEZSIS3DY4X567LBCWAYS77","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a56e2653462936670ce01f4230830d485989d90c274eff3495362a8f56346c8d","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-08T23:30:20Z","title_canon_sha256":"5cf4ced1670542d1a1c6beb9f44ca8063d66f9082d13acc1a9d06ebca7cbd1bd"},"schema_version":"1.0","source":{"id":"2308.04624","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2308.04624","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"arxiv_version","alias_value":"2308.04624v1","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2308.04624","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"pith_short_12","alias_value":"A5VPEZSIS3DY","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"pith_short_16","alias_value":"A5VPEZSIS3DY4X56","created_at":"2026-07-05T06:39:40Z"},{"alias_kind":"pith_short_8","alias_value":"A5VPEZSI","created_at":"2026-07-05T06:39:40Z"}],"graph_snapshots":[{"event_id":"sha256:124c0802355557c630260a122815940246e903efab29b31f5bb50c94e699fa12","target":"graph","created_at":"2026-07-05T06:39:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2308.04624/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Autonomous conversational agents, i.e. chatbots, are becoming an increasingly common mechanism for enterprises to provide support to customers and partners. In order to rate chatbots, especially ones powered by Generative AI tools like Large Language Models (LLMs) we need to be able to accurately assess their performance. This is where chatbot benchmarking becomes important. In this paper, we propose the use of a novel benchmark that we call the E2E (End to End) benchmark, and show how the E2E benchmark can be used to evaluate accuracy and usefulness of the answers provided by chatbots, especi","authors_text":"Arjun Avadhanam, Debarag Banerjee, Pooja Singh, Saksham Srivastava","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-08T23:30:20Z","title":"Benchmarking LLM powered Chatbots: Methods and Metrics"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2308.04624","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f1df8dbc6bf97a4f53158dd544cd4d00f019bf02aaa58284ef39dfc188e74d56","target":"record","created_at":"2026-07-05T06:39:40Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a56e2653462936670ce01f4230830d485989d90c274eff3495362a8f56346c8d","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-08-08T23:30:20Z","title_canon_sha256":"5cf4ced1670542d1a1c6beb9f44ca8063d66f9082d13acc1a9d06ebca7cbd1bd"},"schema_version":"1.0","source":{"id":"2308.04624","kind":"arxiv","version":1}},"canonical_sha256":"076af2664896c78e5fbefac22b0312fffe447758df4e6e3f8fc407ff32ec18fd","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"076af2664896c78e5fbefac22b0312fffe447758df4e6e3f8fc407ff32ec18fd","first_computed_at":"2026-07-05T06:39:40.472374Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T06:39:40.472374Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"nQn/7ZF0tsiexVqQcs51n0YQN12Wg5F9W3mhxVbOlY7BMyWR+8DGbvqy1qrDKfeMvVN2wWPlaNoPJiE9U2JSAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T06:39:40.472844Z","signed_message":"canonical_sha256_bytes"},"source_id":"2308.04624","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f1df8dbc6bf97a4f53158dd544cd4d00f019bf02aaa58284ef39dfc188e74d56","sha256:124c0802355557c630260a122815940246e903efab29b31f5bb50c94e699fa12"],"state_sha256":"bb6758d2fd6dea980a8831572427afb785b5a35908c0f7dac34af110b3a7d50b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ME7vqOaQXMJofFe4QryXvOyTMADcUbg49DgZFxfMILfLuWwDuvjWsVRTE1NmaGFTvP4zrRQg/5HnRUnfDBvlDA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T07:25:44.558008Z","bundle_sha256":"a987e50544035dd621db27092af092afd4999469ee0f53b30a427f8d35824cd0"}}