{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:TCAUJAM2SQATJCJ5DUFTZ7UJGP","short_pith_number":"pith:TCAUJAM2","schema_version":"1.0","canonical_sha256":"988144819a940134893d1d0b3cfe8933f6cb1e441143deedba7b3027803dc1ab","source":{"kind":"arxiv","id":"2411.08275","version":1},"attestation_state":"computed","paper":{"title":"A Large-Scale Study of Relevance Assessments with Large Language Models: An Initial Look","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.IR","authors_text":"Daniel Campos, Hoa Trang Dang, Ian Soboroff, Jimmy Lin, Nandan Thakur, Nick Craswell, Ronak Pradeep, Shivani Upadhyay","submitted_at":"2024-11-13T01:12:35Z","abstract_excerpt":"The application of large language models to provide relevance assessments presents exciting opportunities to advance information retrieval, natural language processing, and beyond, but to date many unknowns remain. This paper reports on the results of a large-scale evaluation (the TREC 2024 RAG Track) where four different relevance assessment approaches were deployed in situ: the \"standard\" fully manual process that NIST has implemented for decades and three different alternatives that take advantage of LLMs to different extents using the open-source UMBRELA tool. This setup allows us to corre"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2411.08275","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2024-11-13T01:12:35Z","cross_cats_sorted":["cs.CL"],"title_canon_sha256":"8bd8ab37ebce9b2a269cb09cecfe99e15e385a0423371bc42d08bfe21d33272a","abstract_canon_sha256":"c5ab18f8ff12993b790aa6d6cf46707c1b36553cacb278ebfa798b1613a1a2cd"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:34:44.537760Z","signature_b64":"s1T7OnAD6WhifBPNH+8nrWFj8szogyXH21JXfYlsd7gpLM+B5XpHXi4KtrouZNolL6ntA/uoaUF469JSL9Q9AA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"988144819a940134893d1d0b3cfe8933f6cb1e441143deedba7b3027803dc1ab","last_reissued_at":"2026-07-05T09:34:44.537281Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:34:44.537281Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"A Large-Scale Study of Relevance Assessments with Large Language Models: An Initial Look","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":["cs.CL"],"primary_cat":"cs.IR","authors_text":"Daniel Campos, Hoa Trang Dang, Ian Soboroff, Jimmy Lin, Nandan Thakur, Nick Craswell, Ronak Pradeep, Shivani Upadhyay","submitted_at":"2024-11-13T01:12:35Z","abstract_excerpt":"The application of large language models to provide relevance assessments presents exciting opportunities to advance information retrieval, natural language processing, and beyond, but to date many unknowns remain. This paper reports on the results of a large-scale evaluation (the TREC 2024 RAG Track) where four different relevance assessment approaches were deployed in situ: the \"standard\" fully manual process that NIST has implemented for decades and three different alternatives that take advantage of LLMs to different extents using the open-source UMBRELA tool. This setup allows us to corre"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2411.08275","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2411.08275/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2411.08275","created_at":"2026-07-05T09:34:44.537340+00:00"},{"alias_kind":"arxiv_version","alias_value":"2411.08275v1","created_at":"2026-07-05T09:34:44.537340+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2411.08275","created_at":"2026-07-05T09:34:44.537340+00:00"},{"alias_kind":"pith_short_12","alias_value":"TCAUJAM2SQAT","created_at":"2026-07-05T09:34:44.537340+00:00"},{"alias_kind":"pith_short_16","alias_value":"TCAUJAM2SQATJCJ5","created_at":"2026-07-05T09:34:44.537340+00:00"},{"alias_kind":"pith_short_8","alias_value":"TCAUJAM2","created_at":"2026-07-05T09:34:44.537340+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":7,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.17634","citing_title":"Prompt Perturbation for Reliable LLM Evaluation over Comparison Graphs","ref_index":17,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04458","citing_title":"DoGMaTiQ: Automated Generation of Question-and-Answer Nuggets for Report Evaluation","ref_index":30,"is_internal_anchor":false},{"citing_arxiv_id":"2605.26400","citing_title":"Plans for Evaluating Structured Generative Search Summaries","ref_index":62,"is_internal_anchor":false},{"citing_arxiv_id":"2602.08457","citing_title":"Hybrid Pooling with LLMs via Relevance Context Learning","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2602.17170","citing_title":"When LLM Judges Inflate Scores: Exploring Overrating in Relevance Assessment","ref_index":26,"is_internal_anchor":false},{"citing_arxiv_id":"2604.04140","citing_title":"Formalized Information Needs Improve Large-Language-Model Relevance Judgments","ref_index":41,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04458","citing_title":"DoGMaTiQ: Automated Generation of Question-and-Answer Nuggets for Report Evaluation","ref_index":30,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP","json":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP.json","graph_json":"https://pith.science/api/pith-number/TCAUJAM2SQATJCJ5DUFTZ7UJGP/graph.json","events_json":"https://pith.science/api/pith-number/TCAUJAM2SQATJCJ5DUFTZ7UJGP/events.json","paper":"https://pith.science/paper/TCAUJAM2"},"agent_actions":{"view_html":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP","download_json":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP.json","view_paper":"https://pith.science/paper/TCAUJAM2","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2411.08275&json=true","fetch_graph":"https://pith.science/api/pith-number/TCAUJAM2SQATJCJ5DUFTZ7UJGP/graph.json","fetch_events":"https://pith.science/api/pith-number/TCAUJAM2SQATJCJ5DUFTZ7UJGP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP/action/storage_attestation","attest_author":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP/action/author_attestation","sign_citation":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP/action/citation_signature","submit_replication":"https://pith.science/pith/TCAUJAM2SQATJCJ5DUFTZ7UJGP/action/replication_record"}},"created_at":"2026-07-05T09:34:44.537340+00:00","updated_at":"2026-07-05T09:34:44.537340+00:00"}