{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:33XBP276VFZJMSPPKJWYPHMYZT","short_pith_number":"pith:33XBP276","canonical_record":{"source":{"id":"2503.13038","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-03-17T10:42:34Z","cross_cats_sorted":[],"title_canon_sha256":"06e423f60cc3936ace5bd3a31431790269d373feea093b108671cc247ab14d98","abstract_canon_sha256":"907a3878540daf380779da2b68300d2fe0ff05b951f364e1fc7f6bf9bbccba3e"},"schema_version":"1.0"},"canonical_sha256":"deee17ebfea9729649ef526d879d98ccdafc4aeb5b49e850a038a951ccab5179","source":{"kind":"arxiv","id":"2503.13038","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.13038","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"arxiv_version","alias_value":"2503.13038v1","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.13038","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"pith_short_12","alias_value":"33XBP276VFZJ","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"pith_short_16","alias_value":"33XBP276VFZJMSPP","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"pith_short_8","alias_value":"33XBP276","created_at":"2026-07-05T10:32:52Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:33XBP276VFZJMSPPKJWYPHMYZT","target":"record","payload":{"canonical_record":{"source":{"id":"2503.13038","kind":"arxiv","version":1},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-03-17T10:42:34Z","cross_cats_sorted":[],"title_canon_sha256":"06e423f60cc3936ace5bd3a31431790269d373feea093b108671cc247ab14d98","abstract_canon_sha256":"907a3878540daf380779da2b68300d2fe0ff05b951f364e1fc7f6bf9bbccba3e"},"schema_version":"1.0"},"canonical_sha256":"deee17ebfea9729649ef526d879d98ccdafc4aeb5b49e850a038a951ccab5179","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:32:52.128536Z","signature_b64":"Tfp/0wgyXrRr4/S1nAj/o3hEFQ4DA4GZnuGq74NNnJmpW+3lk8YE3T8U1oROi/GIMdhAu6IyWFbW62yTtckCBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"deee17ebfea9729649ef526d879d98ccdafc4aeb5b49e850a038a951ccab5179","last_reissued_at":"2026-07-05T10:32:52.128039Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:32:52.128039Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.13038","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:32:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iUNlQTt2V9F3CoEXnAP1vVtcfF/5vObxAwcWKEzS2ZaLBbomABuTHfZRZRHoJBnX94ljIDaif/nsixcAv39/AQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T08:39:26.470945Z"},"content_sha256":"cf817d6079ffd86af3967f958e811db8af4155b9b8ceee23bf1d4deb7e198b6d","schema_version":"1.0","event_id":"sha256:cf817d6079ffd86af3967f958e811db8af4155b9b8ceee23bf1d4deb7e198b6d"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:33XBP276VFZJMSPPKJWYPHMYZT","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Overview of the NTCIR-18 Automatic Evaluation of LLMs (AEOLLM) Task","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Haitao Li, Junjie Chen, Qingyao Ai, Yiqun Liu, Zhumin Chu","submitted_at":"2025-03-17T10:42:34Z","abstract_excerpt":"In this paper, we provide an overview of the NTCIR-18 Automatic Evaluation of LLMs (AEOLLM) task. As large language models (LLMs) grow popular in both academia and industry, how to effectively evaluate the capacity of LLMs becomes an increasingly critical but still challenging issue. Existing methods can be divided into two types: manual evaluation, which is expensive, and automatic evaluation, which faces many limitations including task format (the majority belong to multiple-choice questions) and evaluation criteria (occupied by reference-based metrics). To advance the innovation of automati"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.13038","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.13038/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:32:52Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ngvdJROHeAluoO0J+pgPF7TXoKWqwjIWxCNkrkEk+bNAFrTaqY+p7Zx0REevC+bB05wej4dXes6VtSGKsH20CQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T08:39:26.471330Z"},"content_sha256":"0d1f9f7ad97f0c5636c58c451d4f8ab7157d76c303fe611cff3fde6869ebf170","schema_version":"1.0","event_id":"sha256:0d1f9f7ad97f0c5636c58c451d4f8ab7157d76c303fe611cff3fde6869ebf170"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/33XBP276VFZJMSPPKJWYPHMYZT/bundle.json","state_url":"https://pith.science/pith/33XBP276VFZJMSPPKJWYPHMYZT/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/33XBP276VFZJMSPPKJWYPHMYZT/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T08:39:26Z","links":{"resolver":"https://pith.science/pith/33XBP276VFZJMSPPKJWYPHMYZT","bundle":"https://pith.science/pith/33XBP276VFZJMSPPKJWYPHMYZT/bundle.json","state":"https://pith.science/pith/33XBP276VFZJMSPPKJWYPHMYZT/state.json","well_known_bundle":"https://pith.science/.well-known/pith/33XBP276VFZJMSPPKJWYPHMYZT/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:33XBP276VFZJMSPPKJWYPHMYZT","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"907a3878540daf380779da2b68300d2fe0ff05b951f364e1fc7f6bf9bbccba3e","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-03-17T10:42:34Z","title_canon_sha256":"06e423f60cc3936ace5bd3a31431790269d373feea093b108671cc247ab14d98"},"schema_version":"1.0","source":{"id":"2503.13038","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.13038","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"arxiv_version","alias_value":"2503.13038v1","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.13038","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"pith_short_12","alias_value":"33XBP276VFZJ","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"pith_short_16","alias_value":"33XBP276VFZJMSPP","created_at":"2026-07-05T10:32:52Z"},{"alias_kind":"pith_short_8","alias_value":"33XBP276","created_at":"2026-07-05T10:32:52Z"}],"graph_snapshots":[{"event_id":"sha256:0d1f9f7ad97f0c5636c58c451d4f8ab7157d76c303fe611cff3fde6869ebf170","target":"graph","created_at":"2026-07-05T10:32:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.13038/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"In this paper, we provide an overview of the NTCIR-18 Automatic Evaluation of LLMs (AEOLLM) task. As large language models (LLMs) grow popular in both academia and industry, how to effectively evaluate the capacity of LLMs becomes an increasingly critical but still challenging issue. Existing methods can be divided into two types: manual evaluation, which is expensive, and automatic evaluation, which faces many limitations including task format (the majority belong to multiple-choice questions) and evaluation criteria (occupied by reference-based metrics). To advance the innovation of automati","authors_text":"Haitao Li, Junjie Chen, Qingyao Ai, Yiqun Liu, Zhumin Chu","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-03-17T10:42:34Z","title":"Overview of the NTCIR-18 Automatic Evaluation of LLMs (AEOLLM) Task"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.13038","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:cf817d6079ffd86af3967f958e811db8af4155b9b8ceee23bf1d4deb7e198b6d","target":"record","created_at":"2026-07-05T10:32:52Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"907a3878540daf380779da2b68300d2fe0ff05b951f364e1fc7f6bf9bbccba3e","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2025-03-17T10:42:34Z","title_canon_sha256":"06e423f60cc3936ace5bd3a31431790269d373feea093b108671cc247ab14d98"},"schema_version":"1.0","source":{"id":"2503.13038","kind":"arxiv","version":1}},"canonical_sha256":"deee17ebfea9729649ef526d879d98ccdafc4aeb5b49e850a038a951ccab5179","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"deee17ebfea9729649ef526d879d98ccdafc4aeb5b49e850a038a951ccab5179","first_computed_at":"2026-07-05T10:32:52.128039Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:32:52.128039Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Tfp/0wgyXrRr4/S1nAj/o3hEFQ4DA4GZnuGq74NNnJmpW+3lk8YE3T8U1oROi/GIMdhAu6IyWFbW62yTtckCBw==","signature_status":"signed_v1","signed_at":"2026-07-05T10:32:52.128536Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.13038","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:cf817d6079ffd86af3967f958e811db8af4155b9b8ceee23bf1d4deb7e198b6d","sha256:0d1f9f7ad97f0c5636c58c451d4f8ab7157d76c303fe611cff3fde6869ebf170"],"state_sha256":"a5af7202ca341348dc7f64d8a792f59454e412958b570eabf93a30bc57e45cde"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"i+hsVvhm/FDq6mjayCoKLEZtpOSIIUZv+MUbc4fBbg5RSn2UrWkFDdsNUhk4vQa21ziVPSvsAQPb4U1WgkFeDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T08:39:26.473291Z","bundle_sha256":"e75516f0dc016fd58bc6ce6705b8b48a1fb57ce5b525234bc1fb20770a5bb2d6"}}