{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KP5FLO7M64XOZYDZOONBC3ARNU","short_pith_number":"pith:KP5FLO7M","canonical_record":{"source":{"id":"2406.10421","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T21:52:21Z","cross_cats_sorted":[],"title_canon_sha256":"f0416e4c0f16e41b45ce54e1a552e452e96db1d68726350df77955938d5f100d","abstract_canon_sha256":"a1073bde001098f425725d63154b282287d93f074be9b716dcec8513dd61f057"},"schema_version":"1.0"},"canonical_sha256":"53fa55bbecf72eece079739a116c116d09bb009ad6783fdbed12fae8ceed0604","source":{"kind":"arxiv","id":"2406.10421","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.10421","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"arxiv_version","alias_value":"2406.10421v3","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.10421","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"pith_short_12","alias_value":"KP5FLO7M64XO","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"pith_short_16","alias_value":"KP5FLO7M64XOZYDZ","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"pith_short_8","alias_value":"KP5FLO7M","created_at":"2026-07-05T09:14:34Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KP5FLO7M64XOZYDZOONBC3ARNU","target":"record","payload":{"canonical_record":{"source":{"id":"2406.10421","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T21:52:21Z","cross_cats_sorted":[],"title_canon_sha256":"f0416e4c0f16e41b45ce54e1a552e452e96db1d68726350df77955938d5f100d","abstract_canon_sha256":"a1073bde001098f425725d63154b282287d93f074be9b716dcec8513dd61f057"},"schema_version":"1.0"},"canonical_sha256":"53fa55bbecf72eece079739a116c116d09bb009ad6783fdbed12fae8ceed0604","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:14:34.561691Z","signature_b64":"Va9SfGVmN1RCpBT5+XMrIcixefBTPmi5WNolR6m8r/Nsl/T8vlMsDoGbhnWxd4tswqMJGg48TGipQ/JPkHeHBQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"53fa55bbecf72eece079739a116c116d09bb009ad6783fdbed12fae8ceed0604","last_reissued_at":"2026-07-05T09:14:34.561153Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:14:34.561153Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.10421","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:14:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"AZf6Wzy2qOdLk1drWfAxjv0jT6FE1JF5DG+vdj8MDqoZE22D1C8uimyoY6jOfem0RBqL8bbd93w0Cb7MKw5SCw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T12:58:21.415108Z"},"content_sha256":"b705f81c59c914a5ec112a3d22f5dc14be7030cff05709d0c9fac16dd3495e25","schema_version":"1.0","event_id":"sha256:b705f81c59c914a5ec112a3d22f5dc14be7030cff05709d0c9fac16dd3495e25"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KP5FLO7M64XOZYDZOONBC3ARNU","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"SciEx: Benchmarking Large Language Models on Scientific Exams with Human Expert Grading and Automatic Grading","license":"http://creativecommons.org/licenses/by-sa/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Alexander Waibel, Carlos Mullov, Carsten Dachsbacher, Danni Liu, Fabian Ternava, Jan Niehues, Jianfeng Gao, Jueun Lee, Klemens B\\\"ohm, Leonard B\\\"armann, Michael Beigl, Nathan Lerzer, Rainer Stiefelhagen, Simon Rei{\\ss}, Tamim Asfour, Tobias R\\\"oddiger, Tu Anh Dinh, Zhaolin Li","submitted_at":"2024-06-14T21:52:21Z","abstract_excerpt":"With the rapid development of Large Language Models (LLMs), it is crucial to have benchmarks which can evaluate the ability of LLMs on different domains. One common use of LLMs is performing tasks on scientific topics, such as writing algorithms, querying databases or giving mathematical proofs. Inspired by the way university students are evaluated on such tasks, in this paper, we propose SciEx - a benchmark consisting of university computer science exam questions, to evaluate LLMs ability on solving scientific tasks. SciEx is (1) multilingual, containing both English and German exams, and (2)"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.10421","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.10421/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:14:34Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"YBkmjRSC3/Qri1wtZ4PryiTyxq0zH1W551dWNMhzU939Ibk5lgTxgQTvjzna8D4efzGypg4cLJjrvMlerE7DDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-21T12:58:21.415458Z"},"content_sha256":"f1303c4617e9260797890fead6ed919995eb65e1d14ccbc4da759d2ae50a4e7a","schema_version":"1.0","event_id":"sha256:f1303c4617e9260797890fead6ed919995eb65e1d14ccbc4da759d2ae50a4e7a"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KP5FLO7M64XOZYDZOONBC3ARNU/bundle.json","state_url":"https://pith.science/pith/KP5FLO7M64XOZYDZOONBC3ARNU/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KP5FLO7M64XOZYDZOONBC3ARNU/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-21T12:58:21Z","links":{"resolver":"https://pith.science/pith/KP5FLO7M64XOZYDZOONBC3ARNU","bundle":"https://pith.science/pith/KP5FLO7M64XOZYDZOONBC3ARNU/bundle.json","state":"https://pith.science/pith/KP5FLO7M64XOZYDZOONBC3ARNU/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KP5FLO7M64XOZYDZOONBC3ARNU/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KP5FLO7M64XOZYDZOONBC3ARNU","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"a1073bde001098f425725d63154b282287d93f074be9b716dcec8513dd61f057","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T21:52:21Z","title_canon_sha256":"f0416e4c0f16e41b45ce54e1a552e452e96db1d68726350df77955938d5f100d"},"schema_version":"1.0","source":{"id":"2406.10421","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.10421","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"arxiv_version","alias_value":"2406.10421v3","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.10421","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"pith_short_12","alias_value":"KP5FLO7M64XO","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"pith_short_16","alias_value":"KP5FLO7M64XOZYDZ","created_at":"2026-07-05T09:14:34Z"},{"alias_kind":"pith_short_8","alias_value":"KP5FLO7M","created_at":"2026-07-05T09:14:34Z"}],"graph_snapshots":[{"event_id":"sha256:f1303c4617e9260797890fead6ed919995eb65e1d14ccbc4da759d2ae50a4e7a","target":"graph","created_at":"2026-07-05T09:14:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.10421/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"With the rapid development of Large Language Models (LLMs), it is crucial to have benchmarks which can evaluate the ability of LLMs on different domains. One common use of LLMs is performing tasks on scientific topics, such as writing algorithms, querying databases or giving mathematical proofs. Inspired by the way university students are evaluated on such tasks, in this paper, we propose SciEx - a benchmark consisting of university computer science exam questions, to evaluate LLMs ability on solving scientific tasks. SciEx is (1) multilingual, containing both English and German exams, and (2)","authors_text":"Alexander Waibel, Carlos Mullov, Carsten Dachsbacher, Danni Liu, Fabian Ternava, Jan Niehues, Jianfeng Gao, Jueun Lee, Klemens B\\\"ohm, Leonard B\\\"armann, Michael Beigl, Nathan Lerzer, Rainer Stiefelhagen, Simon Rei{\\ss}, Tamim Asfour, Tobias R\\\"oddiger, Tu Anh Dinh, Zhaolin Li","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T21:52:21Z","title":"SciEx: Benchmarking Large Language Models on Scientific Exams with Human Expert Grading and Automatic Grading"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.10421","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:b705f81c59c914a5ec112a3d22f5dc14be7030cff05709d0c9fac16dd3495e25","target":"record","created_at":"2026-07-05T09:14:34Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"a1073bde001098f425725d63154b282287d93f074be9b716dcec8513dd61f057","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by-sa/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-14T21:52:21Z","title_canon_sha256":"f0416e4c0f16e41b45ce54e1a552e452e96db1d68726350df77955938d5f100d"},"schema_version":"1.0","source":{"id":"2406.10421","kind":"arxiv","version":3}},"canonical_sha256":"53fa55bbecf72eece079739a116c116d09bb009ad6783fdbed12fae8ceed0604","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"53fa55bbecf72eece079739a116c116d09bb009ad6783fdbed12fae8ceed0604","first_computed_at":"2026-07-05T09:14:34.561153Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:14:34.561153Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Va9SfGVmN1RCpBT5+XMrIcixefBTPmi5WNolR6m8r/Nsl/T8vlMsDoGbhnWxd4tswqMJGg48TGipQ/JPkHeHBQ==","signature_status":"signed_v1","signed_at":"2026-07-05T09:14:34.561691Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.10421","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:b705f81c59c914a5ec112a3d22f5dc14be7030cff05709d0c9fac16dd3495e25","sha256:f1303c4617e9260797890fead6ed919995eb65e1d14ccbc4da759d2ae50a4e7a"],"state_sha256":"445d393e33d4fc446e57e90ec3d492d8e1da9ad32963a53589be5c99c3b7a983"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"nO56tvIdNlAEiT+ZvuXAFiPyAI/eeXivv1Ja/mA8oG8W2DWXB0emyh+WPiMFzpduB4ELKsVHnqe20mnmC/HEBw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-21T12:58:21.419388Z","bundle_sha256":"339521ae228f76a170d14cdb103d1ae971c17af61af931a64ba70f8ca4231b66"}}