{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:RNFEI5F32JTHESS2ODC6RZLXOA","short_pith_number":"pith:RNFEI5F3","schema_version":"1.0","canonical_sha256":"8b4a4474bbd266724a5a70c5e8e577702cb9ba03490d52647c9bc9fb3840da88","source":{"kind":"arxiv","id":"2507.19980","version":2},"attestation_state":"computed","paper":{"title":"Exploring LLM Autoscoring Reliability in Large-Scale Writing Assessments Using Generalizability Theory","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dan Song, Hong Jiao, Won-Chan Lee","submitted_at":"2025-07-26T15:33:05Z","abstract_excerpt":"This study investigates the estimation of reliability for large language models (LLMs) in scoring writing tasks from the AP Chinese Language and Culture Exam. Using generalizability theory, the research evaluates and compares score consistency between human and AI raters across two types of AP Chinese free-response writing tasks: story narration and email response. These essays were independently scored by two trained human raters and seven AI raters. Each essay received four scores: one holistic score and three analytic scores corresponding to the domains of task completion, delivery, and lan"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2507.19980","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-07-26T15:33:05Z","cross_cats_sorted":[],"title_canon_sha256":"4add39923b864f7c5da7445a515e1c326db655b1dc94ab558583b85c75bd2917","abstract_canon_sha256":"63ff587bfd50b347a193859a81c3f33fb962e46ef2e5401de1c38d4cd3f96438"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:44:53.214815Z","signature_b64":"4afK8CVv7TEEpwMUtSVoQUN/XUOm0UIGXyulOyZXWgA8NHyGvb3gHzCh6Dbv77FrHkpCQAz+x0fz23o2Tp33Dw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"8b4a4474bbd266724a5a70c5e8e577702cb9ba03490d52647c9bc9fb3840da88","last_reissued_at":"2026-07-05T11:44:53.214388Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:44:53.214388Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Exploring LLM Autoscoring Reliability in Large-Scale Writing Assessments Using Generalizability Theory","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Dan Song, Hong Jiao, Won-Chan Lee","submitted_at":"2025-07-26T15:33:05Z","abstract_excerpt":"This study investigates the estimation of reliability for large language models (LLMs) in scoring writing tasks from the AP Chinese Language and Culture Exam. Using generalizability theory, the research evaluates and compares score consistency between human and AI raters across two types of AP Chinese free-response writing tasks: story narration and email response. These essays were independently scored by two trained human raters and seven AI raters. Each essay received four scores: one holistic score and three analytic scores corresponding to the domains of task completion, delivery, and lan"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.19980","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.19980/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2507.19980","created_at":"2026-07-05T11:44:53.214446+00:00"},{"alias_kind":"arxiv_version","alias_value":"2507.19980v2","created_at":"2026-07-05T11:44:53.214446+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.19980","created_at":"2026-07-05T11:44:53.214446+00:00"},{"alias_kind":"pith_short_12","alias_value":"RNFEI5F32JTH","created_at":"2026-07-05T11:44:53.214446+00:00"},{"alias_kind":"pith_short_16","alias_value":"RNFEI5F32JTHESS2","created_at":"2026-07-05T11:44:53.214446+00:00"},{"alias_kind":"pith_short_8","alias_value":"RNFEI5F3","created_at":"2026-07-05T11:44:53.214446+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2608.11323","citing_title":"Deployment Decision Reliability: A Generalizability-Theory Framework for Sizing Long-Horizon Agent Evaluations","ref_index":12,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA","json":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA.json","graph_json":"https://pith.science/api/pith-number/RNFEI5F32JTHESS2ODC6RZLXOA/graph.json","events_json":"https://pith.science/api/pith-number/RNFEI5F32JTHESS2ODC6RZLXOA/events.json","paper":"https://pith.science/paper/RNFEI5F3"},"agent_actions":{"view_html":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA","download_json":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA.json","view_paper":"https://pith.science/paper/RNFEI5F3","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2507.19980&json=true","fetch_graph":"https://pith.science/api/pith-number/RNFEI5F32JTHESS2ODC6RZLXOA/graph.json","fetch_events":"https://pith.science/api/pith-number/RNFEI5F32JTHESS2ODC6RZLXOA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA/action/storage_attestation","attest_author":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA/action/author_attestation","sign_citation":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA/action/citation_signature","submit_replication":"https://pith.science/pith/RNFEI5F32JTHESS2ODC6RZLXOA/action/replication_record"}},"created_at":"2026-07-05T11:44:53.214446+00:00","updated_at":"2026-07-05T11:44:53.214446+00:00"}