{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:BXW73VLC336QQBBHIHJRTSECFD","short_pith_number":"pith:BXW73VLC","schema_version":"1.0","canonical_sha256":"0dedfdd562defd08042741d319c88228e535cf67e24a75bbca4155fdbba58527","source":{"kind":"arxiv","id":"2505.23836","version":3},"attestation_state":"computed","paper":{"title":"Large Language Models Often Know When They Are Being Evaluated","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Giles Edkins, Govind Pimpale, Henning Bartsch, Joe Needham, Marius Hobbhahn","submitted_at":"2025-05-28T12:03:09Z","abstract_excerpt":"If AI models can detect when they are being evaluated, the effectiveness of evaluations might be compromised. For example, models could have systematically different behavior during evaluations, leading to less reliable benchmarks for deployment and governance decisions. We investigate whether frontier language models can accurately classify transcripts based on whether they originate from evaluations or real-world deployment, a capability we call evaluation awareness. To achieve this, we construct a diverse benchmark of 1,000 prompts and transcripts from 61 distinct datasets. These span publi"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2505.23836","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-28T12:03:09Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"ec586abe6e9851e7efd87f4c7c6731d6e6027b325aeb5e4114f61e2afed59b37","abstract_canon_sha256":"dcf4ee973d75ca4faab27543fa3199bcdca23acdd169a5001059831d4e1bb985"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:38:07.429896Z","signature_b64":"6J+b/eQeSvj3kiaLifMbTiEQTgyCxRG7t3RbPbIeLe7PJ0wB04s5HdUaFSaV1++6GRSl2oW8rE9Tqa4QbNvQCw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0dedfdd562defd08042741d319c88228e535cf67e24a75bbca4155fdbba58527","last_reissued_at":"2026-07-05T11:38:07.429371Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:38:07.429371Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Large Language Models Often Know When They Are Being Evaluated","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Giles Edkins, Govind Pimpale, Henning Bartsch, Joe Needham, Marius Hobbhahn","submitted_at":"2025-05-28T12:03:09Z","abstract_excerpt":"If AI models can detect when they are being evaluated, the effectiveness of evaluations might be compromised. For example, models could have systematically different behavior during evaluations, leading to less reliable benchmarks for deployment and governance decisions. We investigate whether frontier language models can accurately classify transcripts based on whether they originate from evaluations or real-world deployment, a capability we call evaluation awareness. To achieve this, we construct a diverse benchmark of 1,000 prompts and transcripts from 61 distinct datasets. These span publi"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.23836","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.23836/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2505.23836","created_at":"2026-07-05T11:38:07.429440+00:00"},{"alias_kind":"arxiv_version","alias_value":"2505.23836v3","created_at":"2026-07-05T11:38:07.429440+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.23836","created_at":"2026-07-05T11:38:07.429440+00:00"},{"alias_kind":"pith_short_12","alias_value":"BXW73VLC336Q","created_at":"2026-07-05T11:38:07.429440+00:00"},{"alias_kind":"pith_short_16","alias_value":"BXW73VLC336QQBBH","created_at":"2026-07-05T11:38:07.429440+00:00"},{"alias_kind":"pith_short_8","alias_value":"BXW73VLC","created_at":"2026-07-05T11:38:07.429440+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":24,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2607.07184","citing_title":"Predicting LLM Safety Before Release by Simulating Deployment","ref_index":1,"is_internal_anchor":true},{"citing_arxiv_id":"2606.23583","citing_title":"Evaluation Awareness Is Not One Capability: Evidence from Open Language Models","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2606.12731","citing_title":"Normative Robustness as a Frontier for Non-Verifiable Reasoning in LLMs","ref_index":52,"is_internal_anchor":false},{"citing_arxiv_id":"2606.12016","citing_title":"Generalization Hacking: Models Can Game Reinforcement Learning by Preventing Behavioral Generalization","ref_index":8,"is_internal_anchor":false},{"citing_arxiv_id":"2606.11063","citing_title":"CIAware-Bench: Benchmarking Control Intervention Awareness Across Frontier LLMs","ref_index":3,"is_internal_anchor":false},{"citing_arxiv_id":"2606.08629","citing_title":"Sycophancy Towards Researchers Drives Performative Misalignment","ref_index":10,"is_internal_anchor":false},{"citing_arxiv_id":"2606.05391","citing_title":"Human oversight of agentic systems in practice: Examining the oversight work, challenges, and heuristics of developers using software agents","ref_index":88,"is_internal_anchor":false},{"citing_arxiv_id":"2606.02211","citing_title":"Consistency Training while Mitigating Obfuscation via Rate Matching","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2606.07631","citing_title":"Trait-space Monitoring for Emergent Misalignment During Supervised Finetuning","ref_index":51,"is_internal_anchor":false},{"citing_arxiv_id":"2606.31916","citing_title":"Theory of Mind and Persuasion Beyond Conversation: Assessing the Capacity of LLMs to Induce Belief States via Planning and Action","ref_index":54,"is_internal_anchor":false},{"citing_arxiv_id":"2606.28863","citing_title":"Defeat Devices in AI Systems","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2606.06286","citing_title":"LLMs Can Leak Training Data But Do They Want To? A Propensity-Aware Evaluation of Memorization in LLMs","ref_index":34,"is_internal_anchor":false},{"citing_arxiv_id":"2605.23420","citing_title":"Naturalistic measure of social norms alignment","ref_index":5,"is_internal_anchor":false},{"citing_arxiv_id":"2509.18052","citing_title":"The PIMMUR Principles: Ensuring Validity in Collective Behavior of LLM Societies","ref_index":25,"is_internal_anchor":false},{"citing_arxiv_id":"2604.03121","citing_title":"An Independent Safety Evaluation of Kimi K2.5","ref_index":44,"is_internal_anchor":false},{"citing_arxiv_id":"2605.10601","citing_title":"The Open-Box Fallacy: Why AI Deployment Needs a Calibrated Verification Regime","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2604.24966","citing_title":"Risk Reporting for Developers' Internal AI Model Use","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06490","citing_title":"Instrumental Choices: Measuring the Propensity of LLM Agents to Pursue Instrumental Behaviors","ref_index":20,"is_internal_anchor":false},{"citing_arxiv_id":"2605.05835","citing_title":"Evaluation Awareness in Language Models Has Limited Effect on Behaviour","ref_index":19,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02269","citing_title":"Towards Understanding Specification Gaming in Reasoning Models","ref_index":1,"is_internal_anchor":false},{"citing_arxiv_id":"2605.02964","citing_title":"Reward Hacking Benchmark: Measuring Exploits in LLM Agents with Tool Use","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2604.05274","citing_title":"Simulating the Evolution of Alignment and Values in Machine Intelligence","ref_index":14,"is_internal_anchor":false},{"citing_arxiv_id":"2604.13301","citing_title":"Honeypot Protocol","ref_index":11,"is_internal_anchor":false},{"citing_arxiv_id":"2605.06652","citing_title":"When No Benchmark Exists: Validating Comparative LLM Safety Scoring Without Ground-Truth Labels","ref_index":32,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD","json":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD.json","graph_json":"https://pith.science/api/pith-number/BXW73VLC336QQBBHIHJRTSECFD/graph.json","events_json":"https://pith.science/api/pith-number/BXW73VLC336QQBBHIHJRTSECFD/events.json","paper":"https://pith.science/paper/BXW73VLC"},"agent_actions":{"view_html":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD","download_json":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD.json","view_paper":"https://pith.science/paper/BXW73VLC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2505.23836&json=true","fetch_graph":"https://pith.science/api/pith-number/BXW73VLC336QQBBHIHJRTSECFD/graph.json","fetch_events":"https://pith.science/api/pith-number/BXW73VLC336QQBBHIHJRTSECFD/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/action/timestamp_anchor","attest_storage":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/action/storage_attestation","attest_author":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/action/author_attestation","sign_citation":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/action/citation_signature","submit_replication":"https://pith.science/pith/BXW73VLC336QQBBHIHJRTSECFD/action/replication_record"}},"created_at":"2026-07-05T11:38:07.429440+00:00","updated_at":"2026-07-05T11:38:07.429440+00:00"}