{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:NGSKDVWGJ7WG3FGCGRMV2SYIWL","short_pith_number":"pith:NGSKDVWG","canonical_record":{"source":{"id":"2506.08626","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-06-10T09:44:03Z","cross_cats_sorted":[],"title_canon_sha256":"208709d37666147e9e683436147c88bcfb0bf5e44165da932056c0fd04b1019c","abstract_canon_sha256":"2c8f67ba36d1f10aae7d23e22ebfeab30a3791b4652da9358cc8c552f26e303f"},"schema_version":"1.0"},"canonical_sha256":"69a4a1d6c64fec6d94c234595d4b08b2f8c078a54390d034eae58aa91af94970","source":{"kind":"arxiv","id":"2506.08626","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.08626","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"arxiv_version","alias_value":"2506.08626v2","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.08626","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"pith_short_12","alias_value":"NGSKDVWGJ7WG","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"pith_short_16","alias_value":"NGSKDVWGJ7WG3FGC","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"pith_short_8","alias_value":"NGSKDVWG","created_at":"2026-07-05T11:19:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:NGSKDVWGJ7WG3FGCGRMV2SYIWL","target":"record","payload":{"canonical_record":{"source":{"id":"2506.08626","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-06-10T09:44:03Z","cross_cats_sorted":[],"title_canon_sha256":"208709d37666147e9e683436147c88bcfb0bf5e44165da932056c0fd04b1019c","abstract_canon_sha256":"2c8f67ba36d1f10aae7d23e22ebfeab30a3791b4652da9358cc8c552f26e303f"},"schema_version":"1.0"},"canonical_sha256":"69a4a1d6c64fec6d94c234595d4b08b2f8c078a54390d034eae58aa91af94970","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:19:29.942024Z","signature_b64":"ldrNcb0fYKTd+FLvT6qzGII0v+RA3HoCR9/T6WDrKD99Gupoj3VfiDiWpXiah3jXlL/3mg7UxuZED/jUTCDxBA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"69a4a1d6c64fec6d94c234595d4b08b2f8c078a54390d034eae58aa91af94970","last_reissued_at":"2026-07-05T11:19:29.941571Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:19:29.941571Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2506.08626","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:19:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"PN2HiyMlDUfLRZEq+eZgXPjyy6AXBQC3PuGhXrc+s6x1vgh0zb9XDS9fCWqj2hnM998uKyq6dGdwt3/HW9pXAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T11:47:38.907176Z"},"content_sha256":"b0deeb8e1ed7f05e481d66e46a41cecbd2f7fe4a201a27a6ff3c135a8c979f7b","schema_version":"1.0","event_id":"sha256:b0deeb8e1ed7f05e481d66e46a41cecbd2f7fe4a201a27a6ff3c135a8c979f7b"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:NGSKDVWGJ7WG3FGCGRMV2SYIWL","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Leveraging LLMs to Evaluate Usefulness of Document","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Dawei Yin, Erhan Zhang, Jiaxin Mao, Jinghan Xuan, Shuaiqiang Wang, Xingzhu Wang, Ying Nie, Yiqun Chen, Yitong Xu, Yucheng Hou","submitted_at":"2025-06-10T09:44:03Z","abstract_excerpt":"The conventional Cranfield paradigm struggles to effectively capture user satisfaction due to its weak correlation between relevance and satisfaction, alongside the high costs of relevance annotation in building test collections. To tackle these issues, our research explores the potential of leveraging large language models (LLMs) to generate multilevel usefulness labels for evaluation. We introduce a new user-centric evaluation framework that integrates users' search context and behavioral data into LLMs. This framework uses a cascading judgment structure designed for multilevel usefulness as"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.08626","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2506.08626/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:19:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mc1tKnVGKUdKzQmGtCGe58Lbahx0YqAFOnt4Y5HquyeX1ULsWmEEHH+5TLdSIKpe8mS6JDXfCv1AWKXfJOxCBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-05T11:47:38.907973Z"},"content_sha256":"b2f3749ba42690ba4e72554796b4ec50826e4e05fbe1d2c1393b023beda949a6","schema_version":"1.0","event_id":"sha256:b2f3749ba42690ba4e72554796b4ec50826e4e05fbe1d2c1393b023beda949a6"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/NGSKDVWGJ7WG3FGCGRMV2SYIWL/bundle.json","state_url":"https://pith.science/pith/NGSKDVWGJ7WG3FGCGRMV2SYIWL/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/NGSKDVWGJ7WG3FGCGRMV2SYIWL/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-05T11:47:38Z","links":{"resolver":"https://pith.science/pith/NGSKDVWGJ7WG3FGCGRMV2SYIWL","bundle":"https://pith.science/pith/NGSKDVWGJ7WG3FGCGRMV2SYIWL/bundle.json","state":"https://pith.science/pith/NGSKDVWGJ7WG3FGCGRMV2SYIWL/state.json","well_known_bundle":"https://pith.science/.well-known/pith/NGSKDVWGJ7WG3FGCGRMV2SYIWL/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:NGSKDVWGJ7WG3FGCGRMV2SYIWL","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"2c8f67ba36d1f10aae7d23e22ebfeab30a3791b4652da9358cc8c552f26e303f","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-06-10T09:44:03Z","title_canon_sha256":"208709d37666147e9e683436147c88bcfb0bf5e44165da932056c0fd04b1019c"},"schema_version":"1.0","source":{"id":"2506.08626","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2506.08626","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"arxiv_version","alias_value":"2506.08626v2","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2506.08626","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"pith_short_12","alias_value":"NGSKDVWGJ7WG","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"pith_short_16","alias_value":"NGSKDVWGJ7WG3FGC","created_at":"2026-07-05T11:19:29Z"},{"alias_kind":"pith_short_8","alias_value":"NGSKDVWG","created_at":"2026-07-05T11:19:29Z"}],"graph_snapshots":[{"event_id":"sha256:b2f3749ba42690ba4e72554796b4ec50826e4e05fbe1d2c1393b023beda949a6","target":"graph","created_at":"2026-07-05T11:19:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2506.08626/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The conventional Cranfield paradigm struggles to effectively capture user satisfaction due to its weak correlation between relevance and satisfaction, alongside the high costs of relevance annotation in building test collections. To tackle these issues, our research explores the potential of leveraging large language models (LLMs) to generate multilevel usefulness labels for evaluation. We introduce a new user-centric evaluation framework that integrates users' search context and behavioral data into LLMs. This framework uses a cascading judgment structure designed for multilevel usefulness as","authors_text":"Dawei Yin, Erhan Zhang, Jiaxin Mao, Jinghan Xuan, Shuaiqiang Wang, Xingzhu Wang, Ying Nie, Yiqun Chen, Yitong Xu, Yucheng Hou","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-06-10T09:44:03Z","title":"Leveraging LLMs to Evaluate Usefulness of Document"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2506.08626","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b0deeb8e1ed7f05e481d66e46a41cecbd2f7fe4a201a27a6ff3c135a8c979f7b","target":"record","created_at":"2026-07-05T11:19:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"2c8f67ba36d1f10aae7d23e22ebfeab30a3791b4652da9358cc8c552f26e303f","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.IR","submitted_at":"2025-06-10T09:44:03Z","title_canon_sha256":"208709d37666147e9e683436147c88bcfb0bf5e44165da932056c0fd04b1019c"},"schema_version":"1.0","source":{"id":"2506.08626","kind":"arxiv","version":2}},"canonical_sha256":"69a4a1d6c64fec6d94c234595d4b08b2f8c078a54390d034eae58aa91af94970","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"69a4a1d6c64fec6d94c234595d4b08b2f8c078a54390d034eae58aa91af94970","first_computed_at":"2026-07-05T11:19:29.941571Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:19:29.941571Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"ldrNcb0fYKTd+FLvT6qzGII0v+RA3HoCR9/T6WDrKD99Gupoj3VfiDiWpXiah3jXlL/3mg7UxuZED/jUTCDxBA==","signature_status":"signed_v1","signed_at":"2026-07-05T11:19:29.942024Z","signed_message":"canonical_sha256_bytes"},"source_id":"2506.08626","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b0deeb8e1ed7f05e481d66e46a41cecbd2f7fe4a201a27a6ff3c135a8c979f7b","sha256:b2f3749ba42690ba4e72554796b4ec50826e4e05fbe1d2c1393b023beda949a6"],"state_sha256":"c4ca12be4de3e982e2ce43bc492866633b453bcad4365ae52027df95e3e0d294"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"d7/drd02YbW+Qk9QqA2utH49EZqTC1EKSJ4mx/v5tWeNgwPtRenO7NPpkt+Aou+pfqsyRyQRIaUV0TdYOTgvDQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-05T11:47:38.913012Z","bundle_sha256":"ba9b1b998fc86c11939087e0a6b63c97bd3a9ebefe4dc269aa2201ed709b6c41"}}