{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:BXW73VLC336QQBBHIHJRTSECFD","short_pith_number":"pith:BXW73VLC","canonical_record":{"source":{"id":"2505.23836","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-28T12:03:09Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ec586abe6e9851e7efd87f4c7c6731d6e6027b325aeb5e4114f61e2afed59b37","abstract_canon_sha256":"dcf4ee973d75ca4faab27543fa3199bcdca23acdd169a5001059831d4e1bb985"},"schema_version":"1.0"},"canonical_sha256":"0dedfdd562defd08042741d319c88228e535cf67e24a75bbca4155fdbba58527","source":{"kind":"arxiv","id":"2505.23836","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.23836","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"arxiv_version","alias_value":"2505.23836v3","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.23836","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"pith_short_12","alias_value":"BXW73VLC336Q","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"pith_short_16","alias_value":"BXW73VLC336QQBBH","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"pith_short_8","alias_value":"BXW73VLC","created_at":"2026-07-05T11:38:07Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:BXW73VLC336QQBBHIHJRTSECFD","target":"record","payload":{"canonical_record":{"source":{"id":"2505.23836","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-28T12:03:09Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ec586abe6e9851e7efd87f4c7c6731d6e6027b325aeb5e4114f61e2afed59b37","abstract_canon_sha256":"dcf4ee973d75ca4faab27543fa3199bcdca23acdd169a5001059831d4e1bb985"},"schema_version":"1.0"},"canonical_sha256":"0dedfdd562defd08042741d319c88228e535cf67e24a75bbca4155fdbba58527","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:38:07.429896Z","signature_b64":"6J+b/eQeSvj3kiaLifMbTiEQTgyCxRG7t3RbPbIeLe7PJ0wB04s5HdUaFSaV1++6GRSl2oW8rE9Tqa4QbNvQCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0dedfdd562defd08042741d319c88228e535cf67e24a75bbca4155fdbba58527","last_reissued_at":"2026-07-05T11:38:07.429371Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:38:07.429371Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.23836","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:38:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"VX3QehkMg95ETdDGbZ3PZ+GyPCOGegNXnU5Q3hOyhDMDjmysZndwSTJ3hfIkATubUhLfg36HLZiB+wgEEz4sAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-18T14:40:45.521762Z"},"content_sha256":"d9a2fa8428aa7a3f4ad190c884ac0278bf191f13e660ba3abf099f800683c08f","schema_version":"1.0","event_id":"sha256:d9a2fa8428aa7a3f4ad190c884ac0278bf191f13e660ba3abf099f800683c08f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:BXW73VLC336QQBBHIHJRTSECFD","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Large Language Models Often Know When They Are Being Evaluated","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Giles Edkins, Govind Pimpale, Henning Bartsch, Joe Needham, Marius Hobbhahn","submitted_at":"2025-05-28T12:03:09Z","abstract_excerpt":"If AI models can detect when they are being evaluated, the effectiveness of evaluations might be compromised. For example, models could have systematically different behavior during evaluations, leading to less reliable benchmarks for deployment and governance decisions. We investigate whether frontier language models can accurately classify transcripts based on whether they originate from evaluations or real-world deployment, a capability we call evaluation awareness. To achieve this, we construct a diverse benchmark of 1,000 prompts and transcripts from 61 distinct datasets. These span publi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.23836","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.23836/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:38:07Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"QNH91pClVHDZPKS2Q61t3kk4RX9CT1AxmySBMdF/+iOlDcYnFTq1K6TsQF/brqokJ7bfybIf8kf2aNSxySetAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-18T14:40:45.522135Z"},"content_sha256":"fae92145d5873411efdcddb7b8740a1e182d0b229640f2d80e832048f6ed8690","schema_version":"1.0","event_id":"sha256:fae92145d5873411efdcddb7b8740a1e182d0b229640f2d80e832048f6ed8690"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/bundle.json","state_url":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BXW73VLC336QQBBHIHJRTSECFD/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-18T14:40:45Z","links":{"resolver":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD","bundle":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/bundle.json","state":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BXW73VLC336QQBBHIHJRTSECFD/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:BXW73VLC336QQBBHIHJRTSECFD","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"dcf4ee973d75ca4faab27543fa3199bcdca23acdd169a5001059831d4e1bb985","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-28T12:03:09Z","title_canon_sha256":"ec586abe6e9851e7efd87f4c7c6731d6e6027b325aeb5e4114f61e2afed59b37"},"schema_version":"1.0","source":{"id":"2505.23836","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.23836","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"arxiv_version","alias_value":"2505.23836v3","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.23836","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"pith_short_12","alias_value":"BXW73VLC336Q","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"pith_short_16","alias_value":"BXW73VLC336QQBBH","created_at":"2026-07-05T11:38:07Z"},{"alias_kind":"pith_short_8","alias_value":"BXW73VLC","created_at":"2026-07-05T11:38:07Z"}],"graph_snapshots":[{"event_id":"sha256:fae92145d5873411efdcddb7b8740a1e182d0b229640f2d80e832048f6ed8690","target":"graph","created_at":"2026-07-05T11:38:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.23836/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"If AI models can detect when they are being evaluated, the effectiveness of evaluations might be compromised. For example, models could have systematically different behavior during evaluations, leading to less reliable benchmarks for deployment and governance decisions. We investigate whether frontier language models can accurately classify transcripts based on whether they originate from evaluations or real-world deployment, a capability we call evaluation awareness. To achieve this, we construct a diverse benchmark of 1,000 prompts and transcripts from 61 distinct datasets. These span publi","authors_text":"Giles Edkins, Govind Pimpale, Henning Bartsch, Joe Needham, Marius Hobbhahn","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-28T12:03:09Z","title":"Large Language Models Often Know When They Are Being Evaluated"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.23836","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:d9a2fa8428aa7a3f4ad190c884ac0278bf191f13e660ba3abf099f800683c08f","target":"record","created_at":"2026-07-05T11:38:07Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"dcf4ee973d75ca4faab27543fa3199bcdca23acdd169a5001059831d4e1bb985","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-28T12:03:09Z","title_canon_sha256":"ec586abe6e9851e7efd87f4c7c6731d6e6027b325aeb5e4114f61e2afed59b37"},"schema_version":"1.0","source":{"id":"2505.23836","kind":"arxiv","version":3}},"canonical_sha256":"0dedfdd562defd08042741d319c88228e535cf67e24a75bbca4155fdbba58527","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0dedfdd562defd08042741d319c88228e535cf67e24a75bbca4155fdbba58527","first_computed_at":"2026-07-05T11:38:07.429371Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:38:07.429371Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"6J+b/eQeSvj3kiaLifMbTiEQTgyCxRG7t3RbPbIeLe7PJ0wB04s5HdUaFSaV1++6GRSl2oW8rE9Tqa4QbNvQCw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:38:07.429896Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.23836","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:d9a2fa8428aa7a3f4ad190c884ac0278bf191f13e660ba3abf099f800683c08f","sha256:fae92145d5873411efdcddb7b8740a1e182d0b229640f2d80e832048f6ed8690"],"state_sha256":"a9bbe082ef7754db6730460f68dce87fcaba19ba63ec16bbe8e3b36f4bc35c02"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"56fhFdw0HlcMnF43vJzb+p6c8zEn7J710hHtY/rASrr3NK7Uyk8yLp+L4UkpF4LKfKIk1fSzKpk7VfXwy0c5Bw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-18T14:40:45.524242Z","bundle_sha256":"0512dd1448f471e9204214c2457a5c9d589475d7866a871dbc5e9fe3704ace47"}}