{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2016:FBEWEU5MRVBVENXXG6HD7Z6TV3","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"80943573def0a23fb8880fae799e66abccaff1fd7acec70aedfa056212a5308d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2016-12-19T11:48:53Z","title_canon_sha256":"51c36e9dcfa98c028696cb4b1013bb9c460ec91f270c108a876719df8e7fd763"},"schema_version":"1.0","source":{"id":"1612.06136","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1612.06136","created_at":"2026-05-18T00:54:45Z"},{"alias_kind":"arxiv_version","alias_value":"1612.06136v1","created_at":"2026-05-18T00:54:45Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1612.06136","created_at":"2026-05-18T00:54:45Z"},{"alias_kind":"pith_short_12","alias_value":"FBEWEU5MRVBV","created_at":"2026-05-18T12:30:15Z"},{"alias_kind":"pith_short_16","alias_value":"FBEWEU5MRVBVENXX","created_at":"2026-05-18T12:30:15Z"},{"alias_kind":"pith_short_8","alias_value":"FBEWEU5M","created_at":"2026-05-18T12:30:15Z"}],"graph_snapshots":[{"event_id":"sha256:1497237c326ea46d9cc278339ec1e49289f9b775cdd7f98fde83bfa8176b33b1","target":"graph","created_at":"2026-05-18T00:54:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"Ranking evaluation metrics are a fundamental element of design and improvement efforts in information retrieval. We observe that most popular metrics disregard information portrayed in the scores used to derive rankings, when available. This may pose a numerical scaling problem, causing an under- or over-estimation of the evaluation depending on the degree of divergence between the scores of ranked items. The purpose of this work is to propose a principled way of quantifying multi-graded relevance judgments of items and enable a more accurate penalization of ordering errors in rankings. We pro","authors_text":"Jo\\~ao Vinagre, Lu\\'is Torgo, Nuno Moniz","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2016-12-19T11:48:53Z","title":"Data-Driven Relevance Judgments for Ranking Evaluation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1612.06136","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:15487d175f7a362d3d6e3cd90eda34a0d7261f5ff25def7064e34559ad6e91da","target":"record","created_at":"2026-05-18T00:54:45Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"80943573def0a23fb8880fae799e66abccaff1fd7acec70aedfa056212a5308d","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-nc-sa/4.0/","primary_cat":"cs.IR","submitted_at":"2016-12-19T11:48:53Z","title_canon_sha256":"51c36e9dcfa98c028696cb4b1013bb9c460ec91f270c108a876719df8e7fd763"},"schema_version":"1.0","source":{"id":"1612.06136","kind":"arxiv","version":1}},"canonical_sha256":"28496253ac8d435236f7378e3fe7d3aec77dcd9d83f7c311214dfdb4295d1462","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"28496253ac8d435236f7378e3fe7d3aec77dcd9d83f7c311214dfdb4295d1462","first_computed_at":"2026-05-18T00:54:45.854707Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-18T00:54:45.854707Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"c+B8jLF1rypIt9PO+tinZYjiMXBRVuIG7EWTQWdEOKrBpob46au2N77T78uMOkJfnGBzAATsuZ4HKe0UfajQBA==","signature_status":"signed_v1","signed_at":"2026-05-18T00:54:45.855108Z","signed_message":"canonical_sha256_bytes"},"source_id":"1612.06136","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:15487d175f7a362d3d6e3cd90eda34a0d7261f5ff25def7064e34559ad6e91da","sha256:1497237c326ea46d9cc278339ec1e49289f9b775cdd7f98fde83bfa8176b33b1"],"state_sha256":"9191d94b243f0799b086c5f0be3ee29c18ca1cb3c273b22bdb0c00e332c989c2"}