{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:KYYF72BCQZHI3D6VJJ6ZMTZ3UC","short_pith_number":"pith:KYYF72BC","canonical_record":{"source":{"id":"2409.04168","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-06T10:09:41Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"7bd8ddd6fd11faa9218c2704909927429326b7b4022cf2edc30d5254906ef59f","abstract_canon_sha256":"dac10de4193b0b4567a98a1a2b8e48460209efb6e8d5820f59964b9552dd259e"},"schema_version":"1.0"},"canonical_sha256":"56305fe822864e8d8fd54a7d964f3ba0b0e437b70d3b99882811cd6fae75e936","source":{"kind":"arxiv","id":"2409.04168","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.04168","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"arxiv_version","alias_value":"2409.04168v2","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.04168","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"pith_short_12","alias_value":"KYYF72BCQZHI","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"pith_short_16","alias_value":"KYYF72BCQZHI3D6V","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"pith_short_8","alias_value":"KYYF72BC","created_at":"2026-07-05T11:01:55Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:KYYF72BCQZHI3D6VJJ6ZMTZ3UC","target":"record","payload":{"canonical_record":{"source":{"id":"2409.04168","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-06T10:09:41Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"7bd8ddd6fd11faa9218c2704909927429326b7b4022cf2edc30d5254906ef59f","abstract_canon_sha256":"dac10de4193b0b4567a98a1a2b8e48460209efb6e8d5820f59964b9552dd259e"},"schema_version":"1.0"},"canonical_sha256":"56305fe822864e8d8fd54a7d964f3ba0b0e437b70d3b99882811cd6fae75e936","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:01:55.582468Z","signature_b64":"3GGcL++b+cboA4Dk8dcD3C+E0OeB+sJBth7Zo2CpAEFYbjd9lPEGBlFYWK6nIbExIchtg838Mzrf9ffzsotwDQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"56305fe822864e8d8fd54a7d964f3ba0b0e437b70d3b99882811cd6fae75e936","last_reissued_at":"2026-07-05T11:01:55.581878Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:01:55.581878Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2409.04168","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:01:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"+Ua1W4gTuninEou/ggQwoD6MT5KocxN6Iil08bBfP+740WjNOVa9kZk8KAd1AeooocKSOI+Dz6l98gW82XM+AA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T09:42:28.015121Z"},"content_sha256":"f569e1e6dd34b6fbd217f7b3d1e8d0fc3ee4273b9163b594dae3d1d6a24877a0","schema_version":"1.0","event_id":"sha256:f569e1e6dd34b6fbd217f7b3d1e8d0fc3ee4273b9163b594dae3d1d6a24877a0"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:KYYF72BCQZHI3D6VJJ6ZMTZ3UC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"From Calculation to Adjudication: Examining LLM judges on Mathematical Reasoning Tasks","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Andreas Stephan, Benjamin Roth, Dawei Zhu, Matthias A{\\ss}enmacher, Xiaoyu Shen","submitted_at":"2024-09-06T10:09:41Z","abstract_excerpt":"To reduce the need for human annotations, large language models (LLMs) have been proposed as judges of the quality of other candidate models. The performance of LLM judges is typically evaluated by measuring the correlation with human judgments on generative tasks such as summarization or machine translation. In contrast, we study LLM judges on mathematical reasoning tasks. These tasks require multi-step reasoning, and the correctness of their solutions is verifiable, enabling a more objective evaluation. We perform a detailed performance analysis and find that easy samples are easy to judge, "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.04168","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2409.04168/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:01:55Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"8pWRzmJFByzihrXyqB5b6FX926jG6fXvAHs7F6k6i/7pA3JQrZ49T3ybvTYnPuFVmciaQ2H+A6kH6mkdQ0ayAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T09:42:28.015605Z"},"content_sha256":"675dd54d30c72e1172321d45bf21d1bda41fc875a18b463d1964dcc5c769a212","schema_version":"1.0","event_id":"sha256:675dd54d30c72e1172321d45bf21d1bda41fc875a18b463d1964dcc5c769a212"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/KYYF72BCQZHI3D6VJJ6ZMTZ3UC/bundle.json","state_url":"https://pith.science/pith/KYYF72BCQZHI3D6VJJ6ZMTZ3UC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/KYYF72BCQZHI3D6VJJ6ZMTZ3UC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-15T09:42:28Z","links":{"resolver":"https://pith.science/pith/KYYF72BCQZHI3D6VJJ6ZMTZ3UC","bundle":"https://pith.science/pith/KYYF72BCQZHI3D6VJJ6ZMTZ3UC/bundle.json","state":"https://pith.science/pith/KYYF72BCQZHI3D6VJJ6ZMTZ3UC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/KYYF72BCQZHI3D6VJJ6ZMTZ3UC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:KYYF72BCQZHI3D6VJJ6ZMTZ3UC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"dac10de4193b0b4567a98a1a2b8e48460209efb6e8d5820f59964b9552dd259e","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-06T10:09:41Z","title_canon_sha256":"7bd8ddd6fd11faa9218c2704909927429326b7b4022cf2edc30d5254906ef59f"},"schema_version":"1.0","source":{"id":"2409.04168","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2409.04168","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"arxiv_version","alias_value":"2409.04168v2","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2409.04168","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"pith_short_12","alias_value":"KYYF72BCQZHI","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"pith_short_16","alias_value":"KYYF72BCQZHI3D6V","created_at":"2026-07-05T11:01:55Z"},{"alias_kind":"pith_short_8","alias_value":"KYYF72BC","created_at":"2026-07-05T11:01:55Z"}],"graph_snapshots":[{"event_id":"sha256:675dd54d30c72e1172321d45bf21d1bda41fc875a18b463d1964dcc5c769a212","target":"graph","created_at":"2026-07-05T11:01:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2409.04168/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"To reduce the need for human annotations, large language models (LLMs) have been proposed as judges of the quality of other candidate models. The performance of LLM judges is typically evaluated by measuring the correlation with human judgments on generative tasks such as summarization or machine translation. In contrast, we study LLM judges on mathematical reasoning tasks. These tasks require multi-step reasoning, and the correctness of their solutions is verifiable, enabling a more objective evaluation. We perform a detailed performance analysis and find that easy samples are easy to judge, ","authors_text":"Andreas Stephan, Benjamin Roth, Dawei Zhu, Matthias A{\\ss}enmacher, Xiaoyu Shen","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-06T10:09:41Z","title":"From Calculation to Adjudication: Examining LLM judges on Mathematical Reasoning Tasks"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2409.04168","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f569e1e6dd34b6fbd217f7b3d1e8d0fc3ee4273b9163b594dae3d1d6a24877a0","target":"record","created_at":"2026-07-05T11:01:55Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"dac10de4193b0b4567a98a1a2b8e48460209efb6e8d5820f59964b9552dd259e","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-09-06T10:09:41Z","title_canon_sha256":"7bd8ddd6fd11faa9218c2704909927429326b7b4022cf2edc30d5254906ef59f"},"schema_version":"1.0","source":{"id":"2409.04168","kind":"arxiv","version":2}},"canonical_sha256":"56305fe822864e8d8fd54a7d964f3ba0b0e437b70d3b99882811cd6fae75e936","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"56305fe822864e8d8fd54a7d964f3ba0b0e437b70d3b99882811cd6fae75e936","first_computed_at":"2026-07-05T11:01:55.581878Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:01:55.581878Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"3GGcL++b+cboA4Dk8dcD3C+E0OeB+sJBth7Zo2CpAEFYbjd9lPEGBlFYWK6nIbExIchtg838Mzrf9ffzsotwDQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:01:55.582468Z","signed_message":"canonical_sha256_bytes"},"source_id":"2409.04168","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f569e1e6dd34b6fbd217f7b3d1e8d0fc3ee4273b9163b594dae3d1d6a24877a0","sha256:675dd54d30c72e1172321d45bf21d1bda41fc875a18b463d1964dcc5c769a212"],"state_sha256":"eb1c599dc06d0be5f4e45482bb4ba60e130580d6dbfbdb42b99a122f3fb5ffc8"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Po9EEE1ayEYNToD+TBEnlfPIZ154BVj7L9CdtUdkYmjH1IuOLn7RNqpPv7zU1OnEkJlpZjiZnrBRqaRdnHb+Dg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-15T09:42:28.019928Z","bundle_sha256":"93945f4d1e298fb04b69e6c850f1f56179085f8b478e4582a5f60b72f1934557"}}