{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:UJJVJ7VEPJJKI4WBCHNJMSKJDB","short_pith_number":"pith:UJJVJ7VE","canonical_record":{"source":{"id":"2404.10200","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-16T00:54:17Z","cross_cats_sorted":[],"title_canon_sha256":"65cda927c0dcd5bd18c452617781718ab16392ba7a97173a874ea796875e6e2a","abstract_canon_sha256":"88dc299458a391377ce85eb10e46480482b5f2723639b726a7740116cf2d29c3"},"schema_version":"1.0"},"canonical_sha256":"a25354fea47a52a472c111da964949186b1ff0ddcb970b3eb68debb05f3ba348","source":{"kind":"arxiv","id":"2404.10200","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.10200","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"arxiv_version","alias_value":"2404.10200v1","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.10200","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"pith_short_12","alias_value":"UJJVJ7VEPJJK","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"pith_short_16","alias_value":"UJJVJ7VEPJJKI4WB","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"pith_short_8","alias_value":"UJJVJ7VE","created_at":"2026-07-05T08:08:26Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:UJJVJ7VEPJJKI4WBCHNJMSKJDB","target":"record","payload":{"canonical_record":{"source":{"id":"2404.10200","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-16T00:54:17Z","cross_cats_sorted":[],"title_canon_sha256":"65cda927c0dcd5bd18c452617781718ab16392ba7a97173a874ea796875e6e2a","abstract_canon_sha256":"88dc299458a391377ce85eb10e46480482b5f2723639b726a7740116cf2d29c3"},"schema_version":"1.0"},"canonical_sha256":"a25354fea47a52a472c111da964949186b1ff0ddcb970b3eb68debb05f3ba348","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:08:26.645662Z","signature_b64":"SBjTj9tQxf/96CCyVvC5C1Ffjqa6BlfY2j9/xbLk6vbBZm9DJUO2JeN111ZNMgw80vG4a/1rxMuTWhBrmrBHDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a25354fea47a52a472c111da964949186b1ff0ddcb970b3eb68debb05f3ba348","last_reissued_at":"2026-07-05T08:08:26.645284Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:08:26.645284Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2404.10200","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:08:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"5Fqe/Z5NSuuljKe4lvX/yJHzEMX13c+Xz4UsZV63fPRqPKaeq/p6YIFZqOv9DQ5edwsOpQc5y5i4fIfGo/s6Cw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:45:16.830606Z"},"content_sha256":"a69cc95958a6482f3cfd4ee0ad9a5f0be7da9be5a13c5588dd7f2dc050a3c9bf","schema_version":"1.0","event_id":"sha256:a69cc95958a6482f3cfd4ee0ad9a5f0be7da9be5a13c5588dd7f2dc050a3c9bf"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:UJJVJ7VEPJJKI4WBCHNJMSKJDB","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"TEL'M: Test and Evaluation of Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.AI","authors_text":"George Cybenko, Joshua Ackerman, Paul Lintilhac","submitted_at":"2024-04-16T00:54:17Z","abstract_excerpt":"Language Models have demonstrated remarkable capabilities on some tasks while failing dramatically on others. The situation has generated considerable interest in understanding and comparing the capabilities of various Language Models (LMs) but those efforts have been largely ad hoc with results that are often little more than anecdotal. This is in stark contrast with testing and evaluation processes used in healthcare, radar signal processing, and other defense areas. In this paper, we describe Test and Evaluation of Language Models (TEL'M) as a principled approach for assessing the value of "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.10200","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2404.10200/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:08:26Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KxrbQfBKtaJOipNtoZPWM9OttPnjhR/HIPA+9uqxItk9CtWovBQ6IJNrkhMY/0YQtFzUjItwLVfinC9DbYJnBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T12:45:16.830985Z"},"content_sha256":"be6da2ae29826ab45b0cdec07310f12634d9ef9b4a92ccccf1cc3e60a9baecc3","schema_version":"1.0","event_id":"sha256:be6da2ae29826ab45b0cdec07310f12634d9ef9b4a92ccccf1cc3e60a9baecc3"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/UJJVJ7VEPJJKI4WBCHNJMSKJDB/bundle.json","state_url":"https://pith.science/pith/UJJVJ7VEPJJKI4WBCHNJMSKJDB/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/UJJVJ7VEPJJKI4WBCHNJMSKJDB/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T12:45:16Z","links":{"resolver":"https://pith.science/pith/UJJVJ7VEPJJKI4WBCHNJMSKJDB","bundle":"https://pith.science/pith/UJJVJ7VEPJJKI4WBCHNJMSKJDB/bundle.json","state":"https://pith.science/pith/UJJVJ7VEPJJKI4WBCHNJMSKJDB/state.json","well_known_bundle":"https://pith.science/.well-known/pith/UJJVJ7VEPJJKI4WBCHNJMSKJDB/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:UJJVJ7VEPJJKI4WBCHNJMSKJDB","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"88dc299458a391377ce85eb10e46480482b5f2723639b726a7740116cf2d29c3","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-16T00:54:17Z","title_canon_sha256":"65cda927c0dcd5bd18c452617781718ab16392ba7a97173a874ea796875e6e2a"},"schema_version":"1.0","source":{"id":"2404.10200","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2404.10200","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"arxiv_version","alias_value":"2404.10200v1","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2404.10200","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"pith_short_12","alias_value":"UJJVJ7VEPJJK","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"pith_short_16","alias_value":"UJJVJ7VEPJJKI4WB","created_at":"2026-07-05T08:08:26Z"},{"alias_kind":"pith_short_8","alias_value":"UJJVJ7VE","created_at":"2026-07-05T08:08:26Z"}],"graph_snapshots":[{"event_id":"sha256:be6da2ae29826ab45b0cdec07310f12634d9ef9b4a92ccccf1cc3e60a9baecc3","target":"graph","created_at":"2026-07-05T08:08:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2404.10200/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Language Models have demonstrated remarkable capabilities on some tasks while failing dramatically on others. The situation has generated considerable interest in understanding and comparing the capabilities of various Language Models (LMs) but those efforts have been largely ad hoc with results that are often little more than anecdotal. This is in stark contrast with testing and evaluation processes used in healthcare, radar signal processing, and other defense areas. In this paper, we describe Test and Evaluation of Language Models (TEL'M) as a principled approach for assessing the value of ","authors_text":"George Cybenko, Joshua Ackerman, Paul Lintilhac","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-16T00:54:17Z","title":"TEL'M: Test and Evaluation of Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2404.10200","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a69cc95958a6482f3cfd4ee0ad9a5f0be7da9be5a13c5588dd7f2dc050a3c9bf","target":"record","created_at":"2026-07-05T08:08:26Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"88dc299458a391377ce85eb10e46480482b5f2723639b726a7740116cf2d29c3","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2024-04-16T00:54:17Z","title_canon_sha256":"65cda927c0dcd5bd18c452617781718ab16392ba7a97173a874ea796875e6e2a"},"schema_version":"1.0","source":{"id":"2404.10200","kind":"arxiv","version":1}},"canonical_sha256":"a25354fea47a52a472c111da964949186b1ff0ddcb970b3eb68debb05f3ba348","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a25354fea47a52a472c111da964949186b1ff0ddcb970b3eb68debb05f3ba348","first_computed_at":"2026-07-05T08:08:26.645284Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:08:26.645284Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"SBjTj9tQxf/96CCyVvC5C1Ffjqa6BlfY2j9/xbLk6vbBZm9DJUO2JeN111ZNMgw80vG4a/1rxMuTWhBrmrBHDg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:08:26.645662Z","signed_message":"canonical_sha256_bytes"},"source_id":"2404.10200","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a69cc95958a6482f3cfd4ee0ad9a5f0be7da9be5a13c5588dd7f2dc050a3c9bf","sha256:be6da2ae29826ab45b0cdec07310f12634d9ef9b4a92ccccf1cc3e60a9baecc3"],"state_sha256":"e4e8b2a7a6a44463b0f7f1642a5cb9974989680dfda2b363a14f92888c8546b8"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"vPycQVsjMJaR9K6vNbAWdNkPcVg8AOwP76M708UpQWrZ7IJGT10bfAhpc1C9KIfnMxqvD8+bEkMqghbjbzAwAA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T12:45:16.832933Z","bundle_sha256":"432da2bb4079e2e0c0b61e2d85d11c0816671f8b0bdb8c55731898f1a7766ca3"}}