{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:ZV6PMYKEWQ2ERB3IMZLJ4FKD26","short_pith_number":"pith:ZV6PMYKE","canonical_record":{"source":{"id":"2502.01683","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-02T06:36:01Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1db39a16fb54fbb137eaf590516d2f6fb0ca4c2daf4a5d2345831eec8bf969ab","abstract_canon_sha256":"e37095abca0ec1ef5860497552553757478f88db66a5e794c9ef09ddb71498f0"},"schema_version":"1.0"},"canonical_sha256":"cd7cf66144b43448876866569e1543d78415d379f31e6fa7a663b0c2e71ec673","source":{"kind":"arxiv","id":"2502.01683","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.01683","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"arxiv_version","alias_value":"2502.01683v1","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.01683","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"pith_short_12","alias_value":"ZV6PMYKEWQ2E","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"pith_short_16","alias_value":"ZV6PMYKEWQ2ERB3I","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"pith_short_8","alias_value":"ZV6PMYKE","created_at":"2026-07-05T10:09:17Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:ZV6PMYKEWQ2ERB3IMZLJ4FKD26","target":"record","payload":{"canonical_record":{"source":{"id":"2502.01683","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-02T06:36:01Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"1db39a16fb54fbb137eaf590516d2f6fb0ca4c2daf4a5d2345831eec8bf969ab","abstract_canon_sha256":"e37095abca0ec1ef5860497552553757478f88db66a5e794c9ef09ddb71498f0"},"schema_version":"1.0"},"canonical_sha256":"cd7cf66144b43448876866569e1543d78415d379f31e6fa7a663b0c2e71ec673","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T10:09:17.626515Z","signature_b64":"bQPfzp/UU/Y0YZzQBcPbr/ED86cs0ppaizV8Vj25/Bdn0nRCW5+7p0nPM7E40PC2699uztRG/97AQcSDCKiFDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"cd7cf66144b43448876866569e1543d78415d379f31e6fa7a663b0c2e71ec673","last_reissued_at":"2026-07-05T10:09:17.625988Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T10:09:17.625988Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2502.01683","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:09:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"iAm9UhW3ufej++lcZiIja3N6BE4Qtf4XVmTpf0dgjzmst7R130+NCl/1LxhDLgr/jm61AvBCIDfABW9eqLGMAg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T08:27:50.657863Z"},"content_sha256":"a14b444a7820f5b1da3b0b6e55395efbd3f2198446e641d75d7e7ad867d828f0","schema_version":"1.0","event_id":"sha256:a14b444a7820f5b1da3b0b6e55395efbd3f2198446e641d75d7e7ad867d828f0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:ZV6PMYKEWQ2ERB3IMZLJ4FKD26","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"LLM-Powered Benchmark Factory: Reliable, Generic, and Efficient","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Boyuan Pan, Chuyi Tan, Jiayi Shi, Kan Li, Peiwen Yuan, Shaoxiong Feng, Xinglin Wang, Yao Hu, Yiwei Li, Yueqi Zhang","submitted_at":"2025-02-02T06:36:01Z","abstract_excerpt":"The rapid advancement of large language models (LLMs) has led to a surge in both model supply and application demands. To facilitate effective matching between them, reliable, generic and efficient benchmark generators are widely needed. However, human annotators are constrained by inefficiency, and current LLM benchmark generators not only lack generalizability but also struggle with limited reliability, as they lack a comprehensive evaluation framework for validation and optimization. To fill this gap, we first propose an automated and unbiased evaluation framework, structured around four di"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.01683","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2502.01683/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T10:09:17Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Rff6T4S+CQqCuNr/Vnr9nt7U/FJaIWYby0r7re0kMi3/8ISL9uebz3GT6/e7B1sZA4wct+iAY6lIZQosvjDsDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T08:27:50.658364Z"},"content_sha256":"05b37bccbfa2877a9f7e352e5214a1199240161df20b0d03618f04b6148db0ed","schema_version":"1.0","event_id":"sha256:05b37bccbfa2877a9f7e352e5214a1199240161df20b0d03618f04b6148db0ed"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/ZV6PMYKEWQ2ERB3IMZLJ4FKD26/bundle.json","state_url":"https://pith.science/pith/ZV6PMYKEWQ2ERB3IMZLJ4FKD26/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/ZV6PMYKEWQ2ERB3IMZLJ4FKD26/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T08:27:50Z","links":{"resolver":"https://pith.science/pith/ZV6PMYKEWQ2ERB3IMZLJ4FKD26","bundle":"https://pith.science/pith/ZV6PMYKEWQ2ERB3IMZLJ4FKD26/bundle.json","state":"https://pith.science/pith/ZV6PMYKEWQ2ERB3IMZLJ4FKD26/state.json","well_known_bundle":"https://pith.science/.well-known/pith/ZV6PMYKEWQ2ERB3IMZLJ4FKD26/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:ZV6PMYKEWQ2ERB3IMZLJ4FKD26","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"e37095abca0ec1ef5860497552553757478f88db66a5e794c9ef09ddb71498f0","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-02T06:36:01Z","title_canon_sha256":"1db39a16fb54fbb137eaf590516d2f6fb0ca4c2daf4a5d2345831eec8bf969ab"},"schema_version":"1.0","source":{"id":"2502.01683","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2502.01683","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"arxiv_version","alias_value":"2502.01683v1","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2502.01683","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"pith_short_12","alias_value":"ZV6PMYKEWQ2E","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"pith_short_16","alias_value":"ZV6PMYKEWQ2ERB3I","created_at":"2026-07-05T10:09:17Z"},{"alias_kind":"pith_short_8","alias_value":"ZV6PMYKE","created_at":"2026-07-05T10:09:17Z"}],"graph_snapshots":[{"event_id":"sha256:05b37bccbfa2877a9f7e352e5214a1199240161df20b0d03618f04b6148db0ed","target":"graph","created_at":"2026-07-05T10:09:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2502.01683/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The rapid advancement of large language models (LLMs) has led to a surge in both model supply and application demands. To facilitate effective matching between them, reliable, generic and efficient benchmark generators are widely needed. However, human annotators are constrained by inefficiency, and current LLM benchmark generators not only lack generalizability but also struggle with limited reliability, as they lack a comprehensive evaluation framework for validation and optimization. To fill this gap, we first propose an automated and unbiased evaluation framework, structured around four di","authors_text":"Boyuan Pan, Chuyi Tan, Jiayi Shi, Kan Li, Peiwen Yuan, Shaoxiong Feng, Xinglin Wang, Yao Hu, Yiwei Li, Yueqi Zhang","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-02T06:36:01Z","title":"LLM-Powered Benchmark Factory: Reliable, Generic, and Efficient"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2502.01683","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:a14b444a7820f5b1da3b0b6e55395efbd3f2198446e641d75d7e7ad867d828f0","target":"record","created_at":"2026-07-05T10:09:17Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"e37095abca0ec1ef5860497552553757478f88db66a5e794c9ef09ddb71498f0","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-02-02T06:36:01Z","title_canon_sha256":"1db39a16fb54fbb137eaf590516d2f6fb0ca4c2daf4a5d2345831eec8bf969ab"},"schema_version":"1.0","source":{"id":"2502.01683","kind":"arxiv","version":1}},"canonical_sha256":"cd7cf66144b43448876866569e1543d78415d379f31e6fa7a663b0c2e71ec673","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"cd7cf66144b43448876866569e1543d78415d379f31e6fa7a663b0c2e71ec673","first_computed_at":"2026-07-05T10:09:17.625988Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T10:09:17.625988Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"bQPfzp/UU/Y0YZzQBcPbr/ED86cs0ppaizV8Vj25/Bdn0nRCW5+7p0nPM7E40PC2699uztRG/97AQcSDCKiFDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T10:09:17.626515Z","signed_message":"canonical_sha256_bytes"},"source_id":"2502.01683","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:a14b444a7820f5b1da3b0b6e55395efbd3f2198446e641d75d7e7ad867d828f0","sha256:05b37bccbfa2877a9f7e352e5214a1199240161df20b0d03618f04b6148db0ed"],"state_sha256":"ea62ca5af2fa561614181f6cc2dcdf10259891dd185d530ccc8cb53375d0b936"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"mo80jvHTuj7pPt3JE0iED7EhLdcsCC3znGXcqqcR7/oYWrya5zyw7nRuS+DiORHvw/bPz92kRyMzNNNF/yX0BA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T08:27:50.662073Z","bundle_sha256":"d51cddef02acce8b86a8dd2838d88c54f0ef86614bbf6bb5881cc594bff9e4dd"}}