{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:VDYJWSNXPSU6ZKPSKR5FWT3NV3","short_pith_number":"pith:VDYJWSNX","canonical_record":{"source":{"id":"2407.07000","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-07-09T16:13:26Z","cross_cats_sorted":["cs.AI","cs.CL","cs.DC"],"title_canon_sha256":"c486f4fbc56aa968f5ac6316dae526489fb51d5a37e0eb4be048a8b83dac4ac5","abstract_canon_sha256":"0c0170567bf6aa280cf7ee6487431658215312c372ecefd683918667defe81d0"},"schema_version":"1.0"},"canonical_sha256":"a8f09b49b77ca9eca9f2547a5b4f6daed54c92ef9bfc3694579c3517d2bfa96c","source":{"kind":"arxiv","id":"2407.07000","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.07000","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"arxiv_version","alias_value":"2407.07000v2","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.07000","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"pith_short_12","alias_value":"VDYJWSNXPSU6","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"pith_short_16","alias_value":"VDYJWSNXPSU6ZKPS","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"pith_short_8","alias_value":"VDYJWSNX","created_at":"2026-07-05T09:01:03Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:VDYJWSNXPSU6ZKPSKR5FWT3NV3","target":"record","payload":{"canonical_record":{"source":{"id":"2407.07000","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-07-09T16:13:26Z","cross_cats_sorted":["cs.AI","cs.CL","cs.DC"],"title_canon_sha256":"c486f4fbc56aa968f5ac6316dae526489fb51d5a37e0eb4be048a8b83dac4ac5","abstract_canon_sha256":"0c0170567bf6aa280cf7ee6487431658215312c372ecefd683918667defe81d0"},"schema_version":"1.0"},"canonical_sha256":"a8f09b49b77ca9eca9f2547a5b4f6daed54c92ef9bfc3694579c3517d2bfa96c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:01:03.782898Z","signature_b64":"x+8CO/nyj8ieZPNm3KuZEMHAPBcMrXTD4dNQh6aWVE3LoYUbI2EjUDBnoT76IaEWvYjrKjP7poHOaWeXl9deBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a8f09b49b77ca9eca9f2547a5b4f6daed54c92ef9bfc3694579c3517d2bfa96c","last_reissued_at":"2026-07-05T09:01:03.782403Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:01:03.782403Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2407.07000","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:01:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RwQIJA5SOgYI/7RRoZyhnrKn6B6lkgy/aQ7Mz0FGv8tYqK2OiUXniVQlclFAJ/7CcKHbY2m6AX4LXlJovTx3Dg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T09:19:39.578517Z"},"content_sha256":"5a75a0aab4b12dacdc7ba48c043bd9fcde56283bce14df874c78c5f89ac14c1e","schema_version":"1.0","event_id":"sha256:5a75a0aab4b12dacdc7ba48c043bd9fcde56283bce14df874c78c5f89ac14c1e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:VDYJWSNXPSU6ZKPSKR5FWT3NV3","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Etalon: Holistic Performance Evaluation Framework for LLM Inference Systems","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.CL","cs.DC"],"primary_cat":"cs.LG","authors_text":"Alexey Tumanov, Amey Agrawal, Anmol Agarwal, Jayashree Mohan, Nipun Kwatra, Nitin Kedia, Ramachandran Ramjee, Souvik Kundu","submitted_at":"2024-07-09T16:13:26Z","abstract_excerpt":"Serving large language models (LLMs) in production can incur substantial costs, which has prompted recent advances in inference system optimizations. Today, these systems are evaluated against conventional latency and throughput metrics (eg. TTFT, TBT, Normalised Latency and TPOT). However, these metrics fail to fully capture the nuances of LLM inference, leading to an incomplete assessment of user-facing performance crucial for real-time applications such as chat and translation. In this paper, we first identify the pitfalls of current performance metrics in evaluating LLM inference systems. "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.07000","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2407.07000/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:01:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ZELnm6/w+hV4Qif5v4lFgKG8Et70VOqSAOJOxdIblqhY6+iDUch+gL+5NJ9E5iT41a9d9k5gACA0wMwCoz5/Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T09:19:39.579102Z"},"content_sha256":"282d140cfbc66254c6f073476b74bce5554db77736aaed96e75e46e5ddb22277","schema_version":"1.0","event_id":"sha256:282d140cfbc66254c6f073476b74bce5554db77736aaed96e75e46e5ddb22277"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VDYJWSNXPSU6ZKPSKR5FWT3NV3/bundle.json","state_url":"https://pith.science/pith/VDYJWSNXPSU6ZKPSKR5FWT3NV3/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VDYJWSNXPSU6ZKPSKR5FWT3NV3/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T09:19:39Z","links":{"resolver":"https://pith.science/pith/VDYJWSNXPSU6ZKPSKR5FWT3NV3","bundle":"https://pith.science/pith/VDYJWSNXPSU6ZKPSKR5FWT3NV3/bundle.json","state":"https://pith.science/pith/VDYJWSNXPSU6ZKPSKR5FWT3NV3/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VDYJWSNXPSU6ZKPSKR5FWT3NV3/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:VDYJWSNXPSU6ZKPSKR5FWT3NV3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"0c0170567bf6aa280cf7ee6487431658215312c372ecefd683918667defe81d0","cross_cats_sorted":["cs.AI","cs.CL","cs.DC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-07-09T16:13:26Z","title_canon_sha256":"c486f4fbc56aa968f5ac6316dae526489fb51d5a37e0eb4be048a8b83dac4ac5"},"schema_version":"1.0","source":{"id":"2407.07000","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2407.07000","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"arxiv_version","alias_value":"2407.07000v2","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2407.07000","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"pith_short_12","alias_value":"VDYJWSNXPSU6","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"pith_short_16","alias_value":"VDYJWSNXPSU6ZKPS","created_at":"2026-07-05T09:01:03Z"},{"alias_kind":"pith_short_8","alias_value":"VDYJWSNX","created_at":"2026-07-05T09:01:03Z"}],"graph_snapshots":[{"event_id":"sha256:282d140cfbc66254c6f073476b74bce5554db77736aaed96e75e46e5ddb22277","target":"graph","created_at":"2026-07-05T09:01:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2407.07000/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Serving large language models (LLMs) in production can incur substantial costs, which has prompted recent advances in inference system optimizations. Today, these systems are evaluated against conventional latency and throughput metrics (eg. TTFT, TBT, Normalised Latency and TPOT). However, these metrics fail to fully capture the nuances of LLM inference, leading to an incomplete assessment of user-facing performance crucial for real-time applications such as chat and translation. In this paper, we first identify the pitfalls of current performance metrics in evaluating LLM inference systems. ","authors_text":"Alexey Tumanov, Amey Agrawal, Anmol Agarwal, Jayashree Mohan, Nipun Kwatra, Nitin Kedia, Ramachandran Ramjee, Souvik Kundu","cross_cats":["cs.AI","cs.CL","cs.DC"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-07-09T16:13:26Z","title":"Etalon: Holistic Performance Evaluation Framework for LLM Inference Systems"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2407.07000","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:5a75a0aab4b12dacdc7ba48c043bd9fcde56283bce14df874c78c5f89ac14c1e","target":"record","created_at":"2026-07-05T09:01:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"0c0170567bf6aa280cf7ee6487431658215312c372ecefd683918667defe81d0","cross_cats_sorted":["cs.AI","cs.CL","cs.DC"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.LG","submitted_at":"2024-07-09T16:13:26Z","title_canon_sha256":"c486f4fbc56aa968f5ac6316dae526489fb51d5a37e0eb4be048a8b83dac4ac5"},"schema_version":"1.0","source":{"id":"2407.07000","kind":"arxiv","version":2}},"canonical_sha256":"a8f09b49b77ca9eca9f2547a5b4f6daed54c92ef9bfc3694579c3517d2bfa96c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a8f09b49b77ca9eca9f2547a5b4f6daed54c92ef9bfc3694579c3517d2bfa96c","first_computed_at":"2026-07-05T09:01:03.782403Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:01:03.782403Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"x+8CO/nyj8ieZPNm3KuZEMHAPBcMrXTD4dNQh6aWVE3LoYUbI2EjUDBnoT76IaEWvYjrKjP7poHOaWeXl9deBw==","signature_status":"signed_v1","signed_at":"2026-07-05T09:01:03.782898Z","signed_message":"canonical_sha256_bytes"},"source_id":"2407.07000","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:5a75a0aab4b12dacdc7ba48c043bd9fcde56283bce14df874c78c5f89ac14c1e","sha256:282d140cfbc66254c6f073476b74bce5554db77736aaed96e75e46e5ddb22277"],"state_sha256":"d86dac1795f0f59507a6f81a3c3ff13315438b9ee5935db3786dc8c6d1b36506"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"JeqLS25Y7x+TODmqs7z2ffnyJV/akQJsIuCn3540loBAOGGxYwhakyooAJc8ai2KiGPEXEDUJbEHhAR77u+OBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T09:19:39.583532Z","bundle_sha256":"9967d54800a5b8885c52c5b8dea3542555d96366fd518112c098bbb1a35c480f"}}