{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:VFS7P4RE7HCG552IZB6J3BDBZ7","short_pith_number":"pith:VFS7P4RE","canonical_record":{"source":{"id":"2309.17167","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2023-09-29T12:04:14Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"aea1f233d4e99aef0aab6fbc893c7b84cf545718f961f7db7a60b475f6c1bf5d","abstract_canon_sha256":"77c6a4950a44cb36ec4f1478c126940d8451ba6b0999678c420191a6376fcdc1"},"schema_version":"1.0"},"canonical_sha256":"a965f7f224f9c46ef748c87c9d8461cfebca259861be89e03c617d5ec87e6e2c","source":{"kind":"arxiv","id":"2309.17167","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.17167","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"arxiv_version","alias_value":"2309.17167v3","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.17167","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"pith_short_12","alias_value":"VFS7P4RE7HCG","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"pith_short_16","alias_value":"VFS7P4RE7HCG552I","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"pith_short_8","alias_value":"VFS7P4RE","created_at":"2026-07-05T07:55:59Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:VFS7P4RE7HCG552IZB6J3BDBZ7","target":"record","payload":{"canonical_record":{"source":{"id":"2309.17167","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2023-09-29T12:04:14Z","cross_cats_sorted":["cs.CL","cs.LG"],"title_canon_sha256":"aea1f233d4e99aef0aab6fbc893c7b84cf545718f961f7db7a60b475f6c1bf5d","abstract_canon_sha256":"77c6a4950a44cb36ec4f1478c126940d8451ba6b0999678c420191a6376fcdc1"},"schema_version":"1.0"},"canonical_sha256":"a965f7f224f9c46ef748c87c9d8461cfebca259861be89e03c617d5ec87e6e2c","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T07:55:59.722457Z","signature_b64":"E0CZNQtJvc34UisHxntM1XkU01aXkrED01X5Q8YBsp0/rmhiF6z2bXGOOxtNW/+oRDsYwHbTf66qwqRczO4CBg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"a965f7f224f9c46ef748c87c9d8461cfebca259861be89e03c617d5ec87e6e2c","last_reissued_at":"2026-07-05T07:55:59.721946Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T07:55:59.721946Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2309.17167","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:55:59Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"68FpLaZWiCNLi2Oc2xB4nePG8zAhePx+84dMHfRKaS8N1isVQeyleU02rM73+B682/EW9tplcONVnOeRiP5NAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T22:24:00.004541Z"},"content_sha256":"41412d8e10c7c55118b6d6d1c93bbd3f993cfa8e1602849d6490f627faf0958f","schema_version":"1.0","event_id":"sha256:41412d8e10c7c55118b6d6d1c93bbd3f993cfa8e1602849d6490f627faf0958f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:VFS7P4RE7HCG552IZB6J3BDBZ7","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"DyVal: Dynamic Evaluation of Large Language Models for Reasoning Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.CL","cs.LG"],"primary_cat":"cs.AI","authors_text":"Diyi Yang, Jiaao Chen, Jindong Wang, Kaijie Zhu, Neil Zhenqiang Gong, Xing Xie","submitted_at":"2023-09-29T12:04:14Z","abstract_excerpt":"Large language models (LLMs) have achieved remarkable performance in various evaluation benchmarks. However, concerns are raised about potential data contamination in their considerable volume of training corpus. Moreover, the static nature and fixed complexity of current benchmarks may inadequately gauge the advancing capabilities of LLMs. In this paper, we introduce DyVal, a general and flexible protocol for dynamic evaluation of LLMs. Based on our framework, we build graph-informed DyVal by leveraging the structural advantage of directed acyclic graphs to dynamically generate evaluation sam"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.17167","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.17167/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T07:55:59Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"TUfdAkoNYvSuvmE00lLd//1QVjSHTP5Qi2bVXEUqr60U9FkVL8COs8X3dotlVCUpZHL9jPQ6u4kOkbCnhzMvDQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-08T22:24:00.005476Z"},"content_sha256":"579fa17181763064b82e20a09ad65df51d0c341c5c5ae955c52a2caafcebaa52","schema_version":"1.0","event_id":"sha256:579fa17181763064b82e20a09ad65df51d0c341c5c5ae955c52a2caafcebaa52"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/VFS7P4RE7HCG552IZB6J3BDBZ7/bundle.json","state_url":"https://pith.science/pith/VFS7P4RE7HCG552IZB6J3BDBZ7/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/VFS7P4RE7HCG552IZB6J3BDBZ7/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-08T22:24:00Z","links":{"resolver":"https://pith.science/pith/VFS7P4RE7HCG552IZB6J3BDBZ7","bundle":"https://pith.science/pith/VFS7P4RE7HCG552IZB6J3BDBZ7/bundle.json","state":"https://pith.science/pith/VFS7P4RE7HCG552IZB6J3BDBZ7/state.json","well_known_bundle":"https://pith.science/.well-known/pith/VFS7P4RE7HCG552IZB6J3BDBZ7/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:VFS7P4RE7HCG552IZB6J3BDBZ7","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"77c6a4950a44cb36ec4f1478c126940d8451ba6b0999678c420191a6376fcdc1","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2023-09-29T12:04:14Z","title_canon_sha256":"aea1f233d4e99aef0aab6fbc893c7b84cf545718f961f7db7a60b475f6c1bf5d"},"schema_version":"1.0","source":{"id":"2309.17167","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2309.17167","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"arxiv_version","alias_value":"2309.17167v3","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.17167","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"pith_short_12","alias_value":"VFS7P4RE7HCG","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"pith_short_16","alias_value":"VFS7P4RE7HCG552I","created_at":"2026-07-05T07:55:59Z"},{"alias_kind":"pith_short_8","alias_value":"VFS7P4RE","created_at":"2026-07-05T07:55:59Z"}],"graph_snapshots":[{"event_id":"sha256:579fa17181763064b82e20a09ad65df51d0c341c5c5ae955c52a2caafcebaa52","target":"graph","created_at":"2026-07-05T07:55:59Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2309.17167/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large language models (LLMs) have achieved remarkable performance in various evaluation benchmarks. However, concerns are raised about potential data contamination in their considerable volume of training corpus. Moreover, the static nature and fixed complexity of current benchmarks may inadequately gauge the advancing capabilities of LLMs. In this paper, we introduce DyVal, a general and flexible protocol for dynamic evaluation of LLMs. Based on our framework, we build graph-informed DyVal by leveraging the structural advantage of directed acyclic graphs to dynamically generate evaluation sam","authors_text":"Diyi Yang, Jiaao Chen, Jindong Wang, Kaijie Zhu, Neil Zhenqiang Gong, Xing Xie","cross_cats":["cs.CL","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2023-09-29T12:04:14Z","title":"DyVal: Dynamic Evaluation of Large Language Models for Reasoning Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.17167","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:41412d8e10c7c55118b6d6d1c93bbd3f993cfa8e1602849d6490f627faf0958f","target":"record","created_at":"2026-07-05T07:55:59Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"77c6a4950a44cb36ec4f1478c126940d8451ba6b0999678c420191a6376fcdc1","cross_cats_sorted":["cs.CL","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.AI","submitted_at":"2023-09-29T12:04:14Z","title_canon_sha256":"aea1f233d4e99aef0aab6fbc893c7b84cf545718f961f7db7a60b475f6c1bf5d"},"schema_version":"1.0","source":{"id":"2309.17167","kind":"arxiv","version":3}},"canonical_sha256":"a965f7f224f9c46ef748c87c9d8461cfebca259861be89e03c617d5ec87e6e2c","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"a965f7f224f9c46ef748c87c9d8461cfebca259861be89e03c617d5ec87e6e2c","first_computed_at":"2026-07-05T07:55:59.721946Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T07:55:59.721946Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"E0CZNQtJvc34UisHxntM1XkU01aXkrED01X5Q8YBsp0/rmhiF6z2bXGOOxtNW/+oRDsYwHbTf66qwqRczO4CBg==","signature_status":"signed_v1","signed_at":"2026-07-05T07:55:59.722457Z","signed_message":"canonical_sha256_bytes"},"source_id":"2309.17167","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:41412d8e10c7c55118b6d6d1c93bbd3f993cfa8e1602849d6490f627faf0958f","sha256:579fa17181763064b82e20a09ad65df51d0c341c5c5ae955c52a2caafcebaa52"],"state_sha256":"7a210bc487dbc0305a3d3e0f6285f595a7150dcbc4f133292891127374f93bee"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ywRoecfGQ4s1gErJv1Atmb3bM3PZ2An5b7h4Ju1XelkpUv0k9LNQ7Ooi13LPCbxbdED0Y/6WDcj00Kal1XiWCA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-08T22:24:00.015588Z","bundle_sha256":"df0d22907d31351ffe4e4e64ad938a9678aee43e16c85aba8caf3b32d8332830"}}