{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:DUR4R3GCLJBU445LRCFNKZVXKT","short_pith_number":"pith:DUR4R3GC","schema_version":"1.0","canonical_sha256":"1d23c8ecc25a434e73ab888ad566b754dca589205a3d79ff48ed3985160b8047","source":{"kind":"arxiv","id":"2501.13957","version":2},"attestation_state":"computed","paper":{"title":"Benchmarking Generative AI for Scoring Medical Student Interviews in Objective Structured Clinical Examinations (OSCEs)","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Colleen Chan, Dennis Shung, Jadon Geathers, Justin Sewell, Niroop Rajashekar, Rene F. Kizilcec, Susannah Cornes, Yann Hicke","submitted_at":"2025-01-21T04:05:45Z","abstract_excerpt":"Objective Structured Clinical Examinations (OSCEs) are widely used to assess medical students' communication skills, but scoring interview-based assessments is time-consuming and potentially subject to human bias. This study explored the potential of large language models (LLMs) to automate OSCE evaluations using the Master Interview Rating Scale (MIRS). We compared the performance of four state-of-the-art LLMs (GPT-4o, Claude 3.5, Llama 3.1, and Gemini 1.5 Pro) in evaluating OSCE transcripts across all 28 items of the MIRS under the conditions of zero-shot, chain-of-thought (CoT), few-shot, a"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2501.13957","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-01-21T04:05:45Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"d90dab5f91a84177aa6f1ade3b83aeb4c2659015d333fa3e5f6b6749cb665e57","abstract_canon_sha256":"a4335fc468f32c0633ceb632b048146f1d970871326a09edc184876aa87553aa"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:03:20.644777Z","signature_b64":"cJehSA4wSQvusifhSW4O56W2JOOwG7oYP2SXQbgL9PbHTRPdWKvuAdH/Z1bNXr2TSUEUKwnIRJDoirJHH5mHAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"1d23c8ecc25a434e73ab888ad566b754dca589205a3d79ff48ed3985160b8047","last_reissued_at":"2026-07-05T11:03:20.644280Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:03:20.644280Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Benchmarking Generative AI for Scoring Medical Student Interviews in Objective Structured Clinical Examinations (OSCEs)","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Colleen Chan, Dennis Shung, Jadon Geathers, Justin Sewell, Niroop Rajashekar, Rene F. Kizilcec, Susannah Cornes, Yann Hicke","submitted_at":"2025-01-21T04:05:45Z","abstract_excerpt":"Objective Structured Clinical Examinations (OSCEs) are widely used to assess medical students' communication skills, but scoring interview-based assessments is time-consuming and potentially subject to human bias. This study explored the potential of large language models (LLMs) to automate OSCE evaluations using the Master Interview Rating Scale (MIRS). We compared the performance of four state-of-the-art LLMs (GPT-4o, Claude 3.5, Llama 3.1, and Gemini 1.5 Pro) in evaluating OSCE transcripts across all 28 items of the MIRS under the conditions of zero-shot, chain-of-thought (CoT), few-shot, a"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2501.13957","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2501.13957/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2501.13957","created_at":"2026-07-05T11:03:20.644341+00:00"},{"alias_kind":"arxiv_version","alias_value":"2501.13957v2","created_at":"2026-07-05T11:03:20.644341+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2501.13957","created_at":"2026-07-05T11:03:20.644341+00:00"},{"alias_kind":"pith_short_12","alias_value":"DUR4R3GCLJBU","created_at":"2026-07-05T11:03:20.644341+00:00"},{"alias_kind":"pith_short_16","alias_value":"DUR4R3GCLJBU445L","created_at":"2026-07-05T11:03:20.644341+00:00"},{"alias_kind":"pith_short_8","alias_value":"DUR4R3GC","created_at":"2026-07-05T11:03:20.644341+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.28526","citing_title":"A French OSCE Dialogue Dataset and Controllable Virtual Patient System for Clinical Training","ref_index":116,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT","json":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT.json","graph_json":"https://pith.science/api/pith-number/DUR4R3GCLJBU445LRCFNKZVXKT/graph.json","events_json":"https://pith.science/api/pith-number/DUR4R3GCLJBU445LRCFNKZVXKT/events.json","paper":"https://pith.science/paper/DUR4R3GC"},"agent_actions":{"view_html":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT","download_json":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT.json","view_paper":"https://pith.science/paper/DUR4R3GC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2501.13957&json=true","fetch_graph":"https://pith.science/api/pith-number/DUR4R3GCLJBU445LRCFNKZVXKT/graph.json","fetch_events":"https://pith.science/api/pith-number/DUR4R3GCLJBU445LRCFNKZVXKT/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT/action/timestamp_anchor","attest_storage":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT/action/storage_attestation","attest_author":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT/action/author_attestation","sign_citation":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT/action/citation_signature","submit_replication":"https://pith.science/pith/DUR4R3GCLJBU445LRCFNKZVXKT/action/replication_record"}},"created_at":"2026-07-05T11:03:20.644341+00:00","updated_at":"2026-07-05T11:03:20.644341+00:00"}