{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:GFLXO3LQROXL4GK6JLRZRSZ2WI","short_pith_number":"pith:GFLXO3LQ","canonical_record":{"source":{"id":"2412.12148","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2024-12-10T21:57:25Z","cross_cats_sorted":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"title_canon_sha256":"b8a8fd299f7b02c7a0be722dc73685d0abd009cd197686e8edd2cd549ff3089c","abstract_canon_sha256":"ba865dd57e95f114555e3721e5aeb9d30f99fa45ebe4ea255c407560887d9836"},"schema_version":"1.0"},"canonical_sha256":"3157776d708baebe195e4ae398cb3ab222cf5b313838c1bdc21849bd5cb9450e","source":{"kind":"arxiv","id":"2412.12148","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.12148","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"arxiv_version","alias_value":"2412.12148v1","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.12148","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_12","alias_value":"GFLXO3LQROXL","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_16","alias_value":"GFLXO3LQROXL4GK6","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_8","alias_value":"GFLXO3LQ","created_at":"2026-07-05T09:50:10Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:GFLXO3LQROXL4GK6JLRZRSZ2WI","target":"record","payload":{"canonical_record":{"source":{"id":"2412.12148","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2024-12-10T21:57:25Z","cross_cats_sorted":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"title_canon_sha256":"b8a8fd299f7b02c7a0be722dc73685d0abd009cd197686e8edd2cd549ff3089c","abstract_canon_sha256":"ba865dd57e95f114555e3721e5aeb9d30f99fa45ebe4ea255c407560887d9836"},"schema_version":"1.0"},"canonical_sha256":"3157776d708baebe195e4ae398cb3ab222cf5b313838c1bdc21849bd5cb9450e","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:50:10.561208Z","signature_b64":"FOIpsUDIHTaHjkggZlj0FNdYJeZFfv0yn5Z1TItJ9tUqaQOlYpA/Z/YnQvcswpy+f9vdC1Y6aOaWvS64tnQWCA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3157776d708baebe195e4ae398cb3ab222cf5b313838c1bdc21849bd5cb9450e","last_reissued_at":"2026-07-05T09:50:10.560758Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:50:10.560758Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2412.12148","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:50:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Xl/OV8ZcCNn9Kv9L0lArr0QdkoOkzBPgU+yDdDj0iDsxkDpqCIw7vy5p0LosnT42yxODjMlhrFS6Y0L1huW5Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T20:20:55.133050Z"},"content_sha256":"3713a13861d2c41d220b1ba9ef6fe9725a59bc01bf3960b4c6892ecd3740fa3a","schema_version":"1.0","event_id":"sha256:3713a13861d2c41d220b1ba9ef6fe9725a59bc01bf3960b4c6892ecd3740fa3a"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:GFLXO3LQROXL4GK6JLRZRSZ2WI","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"How to Choose a Threshold for an Evaluation Metric for Large Language Models","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"primary_cat":"stat.ML","authors_text":"Bhaskarjit Sarmah, Dhagash Mehta, Jingrao Lyu, Mingshu Li, Nathalia Castellanos, Sebastian Frank, Stefano Pasquali","submitted_at":"2024-12-10T21:57:25Z","abstract_excerpt":"To ensure and monitor large language models (LLMs) reliably, various evaluation metrics have been proposed in the literature. However, there is little research on prescribing a methodology to identify a robust threshold on these metrics even though there are many serious implications of an incorrect choice of the thresholds during deployment of the LLMs. Translating the traditional model risk management (MRM) guidelines within regulated industries such as the financial industry, we propose a step-by-step recipe for picking a threshold for a given LLM evaluation metric. We emphasize that such a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.12148","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2412.12148/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:50:10Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Eu6F6hwfVUqAzujjKt4LiIn0sHR4phB/DN0T7mienby2m1skus2lJR1CVF9k7yM6TZEopAG0iueRzEZhoPfhCg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-18T20:20:55.133540Z"},"content_sha256":"5a7d393b4caabf6c52cc461af0224c46b87eaeb32e00c54615e2660b960c592c","schema_version":"1.0","event_id":"sha256:5a7d393b4caabf6c52cc461af0224c46b87eaeb32e00c54615e2660b960c592c"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/GFLXO3LQROXL4GK6JLRZRSZ2WI/bundle.json","state_url":"https://pith.science/pith/GFLXO3LQROXL4GK6JLRZRSZ2WI/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/GFLXO3LQROXL4GK6JLRZRSZ2WI/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-18T20:20:55Z","links":{"resolver":"https://pith.science/pith/GFLXO3LQROXL4GK6JLRZRSZ2WI","bundle":"https://pith.science/pith/GFLXO3LQROXL4GK6JLRZRSZ2WI/bundle.json","state":"https://pith.science/pith/GFLXO3LQROXL4GK6JLRZRSZ2WI/state.json","well_known_bundle":"https://pith.science/.well-known/pith/GFLXO3LQROXL4GK6JLRZRSZ2WI/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:GFLXO3LQROXL4GK6JLRZRSZ2WI","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"ba865dd57e95f114555e3721e5aeb9d30f99fa45ebe4ea255c407560887d9836","cross_cats_sorted":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2024-12-10T21:57:25Z","title_canon_sha256":"b8a8fd299f7b02c7a0be722dc73685d0abd009cd197686e8edd2cd549ff3089c"},"schema_version":"1.0","source":{"id":"2412.12148","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2412.12148","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"arxiv_version","alias_value":"2412.12148v1","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2412.12148","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_12","alias_value":"GFLXO3LQROXL","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_16","alias_value":"GFLXO3LQROXL4GK6","created_at":"2026-07-05T09:50:10Z"},{"alias_kind":"pith_short_8","alias_value":"GFLXO3LQ","created_at":"2026-07-05T09:50:10Z"}],"graph_snapshots":[{"event_id":"sha256:5a7d393b4caabf6c52cc461af0224c46b87eaeb32e00c54615e2660b960c592c","target":"graph","created_at":"2026-07-05T09:50:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2412.12148/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"To ensure and monitor large language models (LLMs) reliably, various evaluation metrics have been proposed in the literature. However, there is little research on prescribing a methodology to identify a robust threshold on these metrics even though there are many serious implications of an incorrect choice of the thresholds during deployment of the LLMs. Translating the traditional model risk management (MRM) guidelines within regulated industries such as the financial industry, we propose a step-by-step recipe for picking a threshold for a given LLM evaluation metric. We emphasize that such a","authors_text":"Bhaskarjit Sarmah, Dhagash Mehta, Jingrao Lyu, Mingshu Li, Nathalia Castellanos, Sebastian Frank, Stefano Pasquali","cross_cats":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2024-12-10T21:57:25Z","title":"How to Choose a Threshold for an Evaluation Metric for Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2412.12148","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:3713a13861d2c41d220b1ba9ef6fe9725a59bc01bf3960b4c6892ecd3740fa3a","target":"record","created_at":"2026-07-05T09:50:10Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"ba865dd57e95f114555e3721e5aeb9d30f99fa45ebe4ea255c407560887d9836","cross_cats_sorted":["cs.CL","cs.LG","q-fin.ST","stat.AP"],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"stat.ML","submitted_at":"2024-12-10T21:57:25Z","title_canon_sha256":"b8a8fd299f7b02c7a0be722dc73685d0abd009cd197686e8edd2cd549ff3089c"},"schema_version":"1.0","source":{"id":"2412.12148","kind":"arxiv","version":1}},"canonical_sha256":"3157776d708baebe195e4ae398cb3ab222cf5b313838c1bdc21849bd5cb9450e","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"3157776d708baebe195e4ae398cb3ab222cf5b313838c1bdc21849bd5cb9450e","first_computed_at":"2026-07-05T09:50:10.560758Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:50:10.560758Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"FOIpsUDIHTaHjkggZlj0FNdYJeZFfv0yn5Z1TItJ9tUqaQOlYpA/Z/YnQvcswpy+f9vdC1Y6aOaWvS64tnQWCA==","signature_status":"signed_v1","signed_at":"2026-07-05T09:50:10.561208Z","signed_message":"canonical_sha256_bytes"},"source_id":"2412.12148","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:3713a13861d2c41d220b1ba9ef6fe9725a59bc01bf3960b4c6892ecd3740fa3a","sha256:5a7d393b4caabf6c52cc461af0224c46b87eaeb32e00c54615e2660b960c592c"],"state_sha256":"a250a91f583317859942109062b5d331c9364ea8e2b03d78a7890430bf25f73d"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"0MGvaBwhWWGKCYYdhndy+lygJdoSgaJHY18H9FbUM4E81xeai1DAgQrfRwPxmVXxpDodGbwKYDUUcojPy70pBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-18T20:20:55.137498Z","bundle_sha256":"59226c4c33b1cc5b4be37869f78d01d72fe28827eef7df35fc7553bc85caf122"}}