{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2019:K5RPQRZFX6MFHZLEJ6XZUQTMD4","short_pith_number":"pith:K5RPQRZF","canonical_record":{"source":{"id":"1903.11272","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2019-03-27T07:21:55Z","cross_cats_sorted":[],"title_canon_sha256":"87b18514c88d60ea7a584196583441c126f3c96fd9f93c024a6dcd5136375671","abstract_canon_sha256":"bcfd6b2b4135f24ef9f761ee021b84874ebb0d1a59d2fbb0f119fab5554a4067"},"schema_version":"1.0"},"canonical_sha256":"5762f84725bf9853e5644faf9a426c1f22f232a98b5aa2012e14d89bc6e0151f","source":{"kind":"arxiv","id":"1903.11272","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1903.11272","created_at":"2026-05-17T23:50:03Z"},{"alias_kind":"arxiv_version","alias_value":"1903.11272v1","created_at":"2026-05-17T23:50:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1903.11272","created_at":"2026-05-17T23:50:03Z"},{"alias_kind":"pith_short_12","alias_value":"K5RPQRZFX6MF","created_at":"2026-05-18T12:33:21Z"},{"alias_kind":"pith_short_16","alias_value":"K5RPQRZFX6MFHZLE","created_at":"2026-05-18T12:33:21Z"},{"alias_kind":"pith_short_8","alias_value":"K5RPQRZF","created_at":"2026-05-18T12:33:21Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2019:K5RPQRZFX6MFHZLEJ6XZUQTMD4","target":"record","payload":{"canonical_record":{"source":{"id":"1903.11272","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2019-03-27T07:21:55Z","cross_cats_sorted":[],"title_canon_sha256":"87b18514c88d60ea7a584196583441c126f3c96fd9f93c024a6dcd5136375671","abstract_canon_sha256":"bcfd6b2b4135f24ef9f761ee021b84874ebb0d1a59d2fbb0f119fab5554a4067"},"schema_version":"1.0"},"canonical_sha256":"5762f84725bf9853e5644faf9a426c1f22f232a98b5aa2012e14d89bc6e0151f","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-17T23:50:03.699970Z","signature_b64":"z4vaF7QVIu84Nd3oBGtIyXfXdV3ctqOCeXB1iTAFIK1udTN9wTl/4XM5vQwXJdIJdtwdzk/s2VBVfffw1zcOAg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"5762f84725bf9853e5644faf9a426c1f22f232a98b5aa2012e14d89bc6e0151f","last_reissued_at":"2026-05-17T23:50:03.699361Z","signature_status":"signed_v1","first_computed_at":"2026-05-17T23:50:03.699361Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"1903.11272","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:50:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"4HaTK4jvJNkFbrGJgojss+e+oCs83CQ2/QaoThCVkISgJB90pKIRN3GYl2xAJwXBBrYLHkx3Ynn0QLSf+PUOBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-30T03:04:27.406559Z"},"content_sha256":"78f96bde02402a111229530ef10966c514e2a56189177f8a7b5608f2b5d23cd1","schema_version":"1.0","event_id":"sha256:78f96bde02402a111229530ef10966c514e2a56189177f8a7b5608f2b5d23cd1"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2019:K5RPQRZFX6MFHZLEJ6XZUQTMD4","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Graded Relevance Assessments and Graded Relevance Measures of NTCIR: A Survey of the First Twenty Years","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.IR","authors_text":"Tetsuya Sakai","submitted_at":"2019-03-27T07:21:55Z","abstract_excerpt":"NTCIR was the first large-scale IR evaluation conference to construct test collections with graded relevance assessments: the NTCIR-1 test collections from 1998 already featured relevant and partially relevant documents. In this paper, I first describe a few graded-relevance measures that originated from NTCIR (and a few variants) which are used across different NTCIR tasks. I then provide a survey on the use of graded relevance assessments and of graded relevance measures in the past NTCIR tasks which primarily tackled ranked retrieval. My survey shows that the majority of the past tasks full"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1903.11272","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-17T23:50:03Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"jxs7JkuXbtQujS/OjIRMPFMrLZIi0wBOeWT7idEYlErpiK/AlYxDzs0FK0npAwrENdV9gWnWhb6+1mCwbwDFCQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-30T03:04:27.407185Z"},"content_sha256":"2471f66c29415400ec22bae19a09c204498fba2121c6f7ec87e58b3f28d92aa2","schema_version":"1.0","event_id":"sha256:2471f66c29415400ec22bae19a09c204498fba2121c6f7ec87e58b3f28d92aa2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/K5RPQRZFX6MFHZLEJ6XZUQTMD4/bundle.json","state_url":"https://pith.science/pith/K5RPQRZFX6MFHZLEJ6XZUQTMD4/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/K5RPQRZFX6MFHZLEJ6XZUQTMD4/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-30T03:04:27Z","links":{"resolver":"https://pith.science/pith/K5RPQRZFX6MFHZLEJ6XZUQTMD4","bundle":"https://pith.science/pith/K5RPQRZFX6MFHZLEJ6XZUQTMD4/bundle.json","state":"https://pith.science/pith/K5RPQRZFX6MFHZLEJ6XZUQTMD4/state.json","well_known_bundle":"https://pith.science/.well-known/pith/K5RPQRZFX6MFHZLEJ6XZUQTMD4/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2019:K5RPQRZFX6MFHZLEJ6XZUQTMD4","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"bcfd6b2b4135f24ef9f761ee021b84874ebb0d1a59d2fbb0f119fab5554a4067","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2019-03-27T07:21:55Z","title_canon_sha256":"87b18514c88d60ea7a584196583441c126f3c96fd9f93c024a6dcd5136375671"},"schema_version":"1.0","source":{"id":"1903.11272","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"1903.11272","created_at":"2026-05-17T23:50:03Z"},{"alias_kind":"arxiv_version","alias_value":"1903.11272v1","created_at":"2026-05-17T23:50:03Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.1903.11272","created_at":"2026-05-17T23:50:03Z"},{"alias_kind":"pith_short_12","alias_value":"K5RPQRZFX6MF","created_at":"2026-05-18T12:33:21Z"},{"alias_kind":"pith_short_16","alias_value":"K5RPQRZFX6MFHZLE","created_at":"2026-05-18T12:33:21Z"},{"alias_kind":"pith_short_8","alias_value":"K5RPQRZF","created_at":"2026-05-18T12:33:21Z"}],"graph_snapshots":[{"event_id":"sha256:2471f66c29415400ec22bae19a09c204498fba2121c6f7ec87e58b3f28d92aa2","target":"graph","created_at":"2026-05-17T23:50:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"paper":{"abstract_excerpt":"NTCIR was the first large-scale IR evaluation conference to construct test collections with graded relevance assessments: the NTCIR-1 test collections from 1998 already featured relevant and partially relevant documents. In this paper, I first describe a few graded-relevance measures that originated from NTCIR (and a few variants) which are used across different NTCIR tasks. I then provide a survey on the use of graded relevance assessments and of graded relevance measures in the past NTCIR tasks which primarily tackled ranked retrieval. My survey shows that the majority of the past tasks full","authors_text":"Tetsuya Sakai","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2019-03-27T07:21:55Z","title":"Graded Relevance Assessments and Graded Relevance Measures of NTCIR: A Survey of the First Twenty Years"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"1903.11272","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:78f96bde02402a111229530ef10966c514e2a56189177f8a7b5608f2b5d23cd1","target":"record","created_at":"2026-05-17T23:50:03Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"bcfd6b2b4135f24ef9f761ee021b84874ebb0d1a59d2fbb0f119fab5554a4067","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.IR","submitted_at":"2019-03-27T07:21:55Z","title_canon_sha256":"87b18514c88d60ea7a584196583441c126f3c96fd9f93c024a6dcd5136375671"},"schema_version":"1.0","source":{"id":"1903.11272","kind":"arxiv","version":1}},"canonical_sha256":"5762f84725bf9853e5644faf9a426c1f22f232a98b5aa2012e14d89bc6e0151f","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"5762f84725bf9853e5644faf9a426c1f22f232a98b5aa2012e14d89bc6e0151f","first_computed_at":"2026-05-17T23:50:03.699361Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-17T23:50:03.699361Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"z4vaF7QVIu84Nd3oBGtIyXfXdV3ctqOCeXB1iTAFIK1udTN9wTl/4XM5vQwXJdIJdtwdzk/s2VBVfffw1zcOAg==","signature_status":"signed_v1","signed_at":"2026-05-17T23:50:03.699970Z","signed_message":"canonical_sha256_bytes"},"source_id":"1903.11272","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:78f96bde02402a111229530ef10966c514e2a56189177f8a7b5608f2b5d23cd1","sha256:2471f66c29415400ec22bae19a09c204498fba2121c6f7ec87e58b3f28d92aa2"],"state_sha256":"ac6213982bfb4350f0ccfbdd3645b3142a07be58aadf0454df6842448bd34c6b"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"gK4qGyFy6HzatQWfyG4ZxxcLJIG1xmTdLwWSd2b4MjCoB6AL72OA78HS+TmYDbx8EfkBJF4qFOcpsD5r3twbBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-30T03:04:27.410454Z","bundle_sha256":"c96f82494faad8cf33688b835948d59b4a406b80c554aa9b8aba362dbb26fc33"}}