{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2026:XGDJHRTJ64UDHWVITIEOLQLLGA","short_pith_number":"pith:XGDJHRTJ","schema_version":"1.0","canonical_sha256":"b98693c669f72833daa89a08e5c16b302b1670d51f3fc3157732c7f0e77576cd","source":{"kind":"arxiv","id":"2607.29252","version":1},"attestation_state":"computed","paper":{"title":"CalibratedRubric: Task-Adaptive Rubric Banks for Open-Ended LLM Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Beidi Luan, Dezhi Chen, Jing Li, Mengting Chen, Rui Sun, Wanting Liang, Yanshu Sun, Zuo Bai","submitted_at":"2026-07-31T10:21:56Z","abstract_excerpt":"Reliable evaluation of open-ended LLM outputs requires fine-grained rubrics, yet expert curation is costly and difficult to scale. Existing automated pipelines rely on strict judge unanimity and binary variance filters, which cannot distinguish measurable rubrics from informative ones. We introduce CalibratedRubric, a task-adaptive framework that combines type-specific scoring, Bayesian rubric-measurability filtering, and item response theory (IRT)-based bank assembly. CalibratedRubric estimates each rubric's measurability with a Beta--Bernoulli agreement posterior and uses a submodular inform"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2607.29252","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2026-07-31T10:21:56Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"4b1e5810efeeb9a07e309e17d04505b4a939b9dd43d1779bdfa908a748c56c86","abstract_canon_sha256":"70901a1f680ac13f5c16053949964f70e1b6407b6fd2de1b78adddb9bf281226"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-08-03T01:22:27.406813Z","signature_b64":"cCXZZqCi4n/eJKnG1lLYZKrZ50KMeB2k33kp/PtITPlQ9g94GB36aPTn2m8aOUwzaxV6tnGdGUPFkhTZ+wyPAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"b98693c669f72833daa89a08e5c16b302b1670d51f3fc3157732c7f0e77576cd","last_reissued_at":"2026-08-03T01:22:27.405116Z","signature_status":"signed_v1","first_computed_at":"2026-08-03T01:22:27.405116Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"CalibratedRubric: Task-Adaptive Rubric Banks for Open-Ended LLM Evaluation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Beidi Luan, Dezhi Chen, Jing Li, Mengting Chen, Rui Sun, Wanting Liang, Yanshu Sun, Zuo Bai","submitted_at":"2026-07-31T10:21:56Z","abstract_excerpt":"Reliable evaluation of open-ended LLM outputs requires fine-grained rubrics, yet expert curation is costly and difficult to scale. Existing automated pipelines rely on strict judge unanimity and binary variance filters, which cannot distinguish measurable rubrics from informative ones. We introduce CalibratedRubric, a task-adaptive framework that combines type-specific scoring, Bayesian rubric-measurability filtering, and item response theory (IRT)-based bank assembly. CalibratedRubric estimates each rubric's measurability with a Beta--Bernoulli agreement posterior and uses a submodular inform"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2607.29252","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2607.29252/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2607.29252","created_at":"2026-08-03T01:22:27.406058+00:00"},{"alias_kind":"arxiv_version","alias_value":"2607.29252v1","created_at":"2026-08-03T01:22:27.406058+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2607.29252","created_at":"2026-08-03T01:22:27.406058+00:00"},{"alias_kind":"pith_short_12","alias_value":"XGDJHRTJ64UD","created_at":"2026-08-03T01:22:27.406058+00:00"},{"alias_kind":"pith_short_16","alias_value":"XGDJHRTJ64UDHWVI","created_at":"2026-08-03T01:22:27.406058+00:00"},{"alias_kind":"pith_short_8","alias_value":"XGDJHRTJ","created_at":"2026-08-03T01:22:27.406058+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA","json":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA.json","graph_json":"https://pith.science/api/pith-number/XGDJHRTJ64UDHWVITIEOLQLLGA/graph.json","events_json":"https://pith.science/api/pith-number/XGDJHRTJ64UDHWVITIEOLQLLGA/events.json","paper":"https://pith.science/paper/XGDJHRTJ"},"agent_actions":{"view_html":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA","download_json":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA.json","view_paper":"https://pith.science/paper/XGDJHRTJ","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2607.29252&json=true","fetch_graph":"https://pith.science/api/pith-number/XGDJHRTJ64UDHWVITIEOLQLLGA/graph.json","fetch_events":"https://pith.science/api/pith-number/XGDJHRTJ64UDHWVITIEOLQLLGA/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA/action/timestamp_anchor","attest_storage":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA/action/storage_attestation","attest_author":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA/action/author_attestation","sign_citation":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA/action/citation_signature","submit_replication":"https://pith.science/pith/XGDJHRTJ64UDHWVITIEOLQLLGA/action/replication_record"}},"created_at":"2026-08-03T01:22:27.406058+00:00","updated_at":"2026-08-03T01:22:27.406058+00:00"}