{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:GEBUETWALA7HSKJ2GDNH4RXHQP","short_pith_number":"pith:GEBUETWA","schema_version":"1.0","canonical_sha256":"3103424ec0583e79293a30da7e46e783eb9d6275dfeab92b2254475550199f1d","source":{"kind":"arxiv","id":"2406.06519","version":1},"attestation_state":"computed","paper":{"title":"UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Jimmy Lin, Nandan Thakur, Nick Craswell, Ronak Pradeep, Shivani Upadhyay","submitted_at":"2024-06-10T17:58:29Z","abstract_excerpt":"Copious amounts of relevance judgments are necessary for the effective training and accurate evaluation of retrieval systems. Conventionally, these judgments are made by human assessors, rendering this process expensive and laborious. A recent study by Thomas et al. from Microsoft Bing suggested that large language models (LLMs) can accurately perform the relevance assessment task and provide human-quality judgments, but unfortunately their study did not yield any reusable software artifacts. Our work presents UMBRELA (a recursive acronym that stands for UMbrela is the Bing RELevance Assessor)"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2406.06519","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2024-06-10T17:58:29Z","cross_cats_sorted":[],"title_canon_sha256":"0d42f1a4c5c993ff0d122df1cc296556ed5780d674ad63ee6fd6bc93e8c468b9","abstract_canon_sha256":"1c92986fa67e0b1662259610c14431bee7df1d02f9662ad0deb974a4af437f77"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:29:43.524347Z","signature_b64":"GyNRza7m1TwV1uQKrUra0UuKSne5Q93NEa4VfpWfIehd8t+z1hGFj+BIYHITyg0dUeFEE9AcIhxjTJeqKzVMBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"3103424ec0583e79293a30da7e46e783eb9d6275dfeab92b2254475550199f1d","last_reissued_at":"2026-07-05T08:29:43.523933Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:29:43.523933Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"UMBRELA: UMbrela is the (Open-Source Reproduction of the) Bing RELevance Assessor","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Jimmy Lin, Nandan Thakur, Nick Craswell, Ronak Pradeep, Shivani Upadhyay","submitted_at":"2024-06-10T17:58:29Z","abstract_excerpt":"Copious amounts of relevance judgments are necessary for the effective training and accurate evaluation of retrieval systems. Conventionally, these judgments are made by human assessors, rendering this process expensive and laborious. A recent study by Thomas et al. from Microsoft Bing suggested that large language models (LLMs) can accurately perform the relevance assessment task and provide human-quality judgments, but unfortunately their study did not yield any reusable software artifacts. Our work presents UMBRELA (a recursive acronym that stands for UMbrela is the Bing RELevance Assessor)"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.06519","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.06519/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2406.06519","created_at":"2026-07-05T08:29:43.523987+00:00"},{"alias_kind":"arxiv_version","alias_value":"2406.06519v1","created_at":"2026-07-05T08:29:43.523987+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.06519","created_at":"2026-07-05T08:29:43.523987+00:00"},{"alias_kind":"pith_short_12","alias_value":"GEBUETWALA7H","created_at":"2026-07-05T08:29:43.523987+00:00"},{"alias_kind":"pith_short_16","alias_value":"GEBUETWALA7HSKJ2","created_at":"2026-07-05T08:29:43.523987+00:00"},{"alias_kind":"pith_short_8","alias_value":"GEBUETWA","created_at":"2026-07-05T08:29:43.523987+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":11,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.29467","citing_title":"mamabench and mamaretrieval: Benchmarks for Evaluating Medical Retrieval-Augmented Generation in Maternal, Neonatal, and Reproductive Health","ref_index":13,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04458","citing_title":"DoGMaTiQ: Automated Generation of Question-and-Answer Nuggets for Report Evaluation","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2606.29467","citing_title":"mamabench and mamaretrieval: Benchmarks for Evaluating Medical Retrieval-Augmented Generation in Maternal, Neonatal, and Reproductive Health","ref_index":12,"is_internal_anchor":false},{"citing_arxiv_id":"2605.28787","citing_title":"Do Agents Need Semantic Metadata? A Comparative Study in Agentic Data Retrieval","ref_index":32,"is_internal_anchor":false},{"citing_arxiv_id":"2606.27559","citing_title":"A Sensitivity-Aware Test Collection for Search Among Personal Information","ref_index":64,"is_internal_anchor":false},{"citing_arxiv_id":"2602.08457","citing_title":"Hybrid Pooling with LLMs via Relevance Context Learning","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2602.17170","citing_title":"When LLM Judges Inflate Scores: Exploring Overrating in Relevance Assessment","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2604.04140","citing_title":"Formalized Information Needs Improve Large-Language-Model Relevance Judgments","ref_index":42,"is_internal_anchor":false},{"citing_arxiv_id":"2605.04458","citing_title":"DoGMaTiQ: Automated Generation of Question-and-Answer Nuggets for Report Evaluation","ref_index":31,"is_internal_anchor":false},{"citing_arxiv_id":"2605.01591","citing_title":"Led to Mislead: Adversarial Content Injection for Attacks on Neural Ranking Models","ref_index":50,"is_internal_anchor":false},{"citing_arxiv_id":"2604.09492","citing_title":"Dynamic Ranked List Truncation for Reranking Pipelines via LLM-generated Reference-Documents","ref_index":38,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP","json":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP.json","graph_json":"https://pith.science/api/pith-number/GEBUETWALA7HSKJ2GDNH4RXHQP/graph.json","events_json":"https://pith.science/api/pith-number/GEBUETWALA7HSKJ2GDNH4RXHQP/events.json","paper":"https://pith.science/paper/GEBUETWA"},"agent_actions":{"view_html":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP","download_json":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP.json","view_paper":"https://pith.science/paper/GEBUETWA","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2406.06519&json=true","fetch_graph":"https://pith.science/api/pith-number/GEBUETWALA7HSKJ2GDNH4RXHQP/graph.json","fetch_events":"https://pith.science/api/pith-number/GEBUETWALA7HSKJ2GDNH4RXHQP/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP/action/timestamp_anchor","attest_storage":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP/action/storage_attestation","attest_author":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP/action/author_attestation","sign_citation":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP/action/citation_signature","submit_replication":"https://pith.science/pith/GEBUETWALA7HSKJ2GDNH4RXHQP/action/replication_record"}},"created_at":"2026-07-05T08:29:43.523987+00:00","updated_at":"2026-07-05T08:29:43.523987+00:00"}