{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:I44TW35NPR5PJAAZBXKBHS6KEN","short_pith_number":"pith:I44TW35N","canonical_record":{"source":{"id":"2512.00920","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-11-30T14:54:12Z","cross_cats_sorted":[],"title_canon_sha256":"58a3a4d390f8261f81cba59cb1c194c6484e3151b0a82d6103dee92fb921eba3","abstract_canon_sha256":"34c28eddc3a84ec52591111031a61ace34d1728a5b1e5a47f6dd1f16ec7690d0"},"schema_version":"1.0"},"canonical_sha256":"47393b6fad7c7af480190dd413cbca234016c43855b6f69389c34c9d00954930","source":{"kind":"arxiv","id":"2512.00920","version":5},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2512.00920","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"arxiv_version","alias_value":"2512.00920v5","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2512.00920","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"pith_short_12","alias_value":"I44TW35NPR5P","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"pith_short_16","alias_value":"I44TW35NPR5PJAAZ","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"pith_short_8","alias_value":"I44TW35N","created_at":"2026-05-20T00:00:29Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:I44TW35NPR5PJAAZBXKBHS6KEN","target":"record","payload":{"canonical_record":{"source":{"id":"2512.00920","kind":"arxiv","version":5},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-11-30T14:54:12Z","cross_cats_sorted":[],"title_canon_sha256":"58a3a4d390f8261f81cba59cb1c194c6484e3151b0a82d6103dee92fb921eba3","abstract_canon_sha256":"34c28eddc3a84ec52591111031a61ace34d1728a5b1e5a47f6dd1f16ec7690d0"},"schema_version":"1.0"},"canonical_sha256":"47393b6fad7c7af480190dd413cbca234016c43855b6f69389c34c9d00954930","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-05-20T00:00:29.145123Z","signature_b64":"SpbvHbNk5O/K6MNokjzhNnK92konFYZcgUPr2Z9z/XpBWVnC9ZsVUo7GZyi5JB/brAOdxv2EnkzqTcIAwm7hDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"47393b6fad7c7af480190dd413cbca234016c43855b6f69389c34c9d00954930","last_reissued_at":"2026-05-20T00:00:29.144435Z","signature_status":"signed_v1","first_computed_at":"2026-05-20T00:00:29.144435Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2512.00920","source_version":5,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-20T00:00:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"b0W4OEES61inZfQ3xqNYTP2P1+I5heyRIRHgmbz7dXTBvBeUNUytP3u4chDOZT3qKfm8/7MkD6xY4TxB9OIBDw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-10T22:55:33.536170Z"},"content_sha256":"464e48d6fdf6c3788a826658fc652f2c1bd2f6b0dcd42b6b969d2bf403515e31","schema_version":"1.0","event_id":"sha256:464e48d6fdf6c3788a826658fc652f2c1bd2f6b0dcd42b6b969d2bf403515e31"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:I44TW35NPR5PJAAZBXKBHS6KEN","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Reward Auditor: Inference on Reward Modeling Suitability in Real-World Perturbed Scenarios","license":"http://creativecommons.org/licenses/by/4.0/","headline":"Reward Auditor uses hypothesis testing to detect if reward models have systematic vulnerabilities under real-world perturbations.","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Binhong Li, Hui Liu, Jianxiang Zang, Nijia Mo, Qiang Sun, Ruxue Bai, Shiyu Jiang, Yongda Wei","submitted_at":"2025-11-30T14:54:12Z","abstract_excerpt":"Reliable reward models (RMs) are critical for ensuring the safe alignment of large language models (LLMs). However, current RM evaluation methods focus solely on preference perception accuracies in given specific scenarios, obscuring the critical vulnerabilities of RMs in real-world scenarios. We identify the true challenge lies in assessing a novel dimension: Suitability, defined as conditional reliability under specific real-world perturbations. To this end, we introduce Reward Auditor, a hypothesis-testing framework specifically designed for RM suitability inference. Rather than answering \""},"claims":{"count":4,"items":[{"kind":"strongest_claim","text":"Under real-world perturbed scenarios, Reward Auditor quantifies statistical significance and effect size by auditing distribution degradation of RM preference perception confidence. This enables inference of both the certainty and severity of RM vulnerabilities across diverse real-world scenarios.","source":"verdict.strongest_claim","status":"machine_extracted","claim_id":"C1","attestation":"unclaimed"},{"kind":"weakest_assumption","text":"The chosen real-world perturbations and the definition of suitability as conditional reliability under those perturbations accurately capture the vulnerabilities that matter for safe LLM alignment in deployment.","source":"verdict.weakest_assumption","status":"machine_extracted","claim_id":"C2","attestation":"unclaimed"},{"kind":"one_line_summary","text":"Reward Auditor is a statistical auditing framework that infers systematic vulnerabilities in reward models by quantifying distribution degradation of preference perception confidence under real-world perturbations.","source":"verdict.one_line_summary","status":"machine_extracted","claim_id":"C3","attestation":"unclaimed"},{"kind":"headline","text":"Reward Auditor uses hypothesis testing to detect if reward models have systematic vulnerabilities under real-world perturbations.","source":"verdict.pith_extraction.headline","status":"machine_extracted","claim_id":"C4","attestation":"unclaimed"}],"snapshot_sha256":"eafe369da999e7fbc1213aa5908723cf0856322c49bd8e9a0638dde7e93c3b8f"},"source":{"id":"2512.00920","kind":"arxiv","version":5},"verdict":{"id":"998e36d9-10c0-4cf3-8cf6-b8ed7d67288d","model_set":{"reader":"grok-4.3"},"created_at":"2026-05-17T02:53:18.202770Z","strongest_claim":"Under real-world perturbed scenarios, Reward Auditor quantifies statistical significance and effect size by auditing distribution degradation of RM preference perception confidence. This enables inference of both the certainty and severity of RM vulnerabilities across diverse real-world scenarios.","one_line_summary":"Reward Auditor is a statistical auditing framework that infers systematic vulnerabilities in reward models by quantifying distribution degradation of preference perception confidence under real-world perturbations.","pipeline_version":"pith-pipeline@v0.9.0","weakest_assumption":"The chosen real-world perturbations and the definition of suitability as conditional reliability under those perturbations accurately capture the vulnerabilities that matter for safe LLM alignment in deployment.","pith_extraction_headline":"Reward Auditor uses hypothesis testing to detect if reward models have systematic vulnerabilities under real-world perturbations."},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2512.00920/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":"998e36d9-10c0-4cf3-8cf6-b8ed7d67288d"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-05-20T00:00:29Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"GUUPsQQNpzO3omuwHkqlUULfWRPVU9TaYbC2BmV8irwscQvmMrGGxhanPHZO1npiFhxiCkpSiom+n14QRwktAA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-06-10T22:55:33.536654Z"},"content_sha256":"bc5ebef07fc3eadb629fb191efb7d208582dc5f5a6d1688f6ee0f946af7635c2","schema_version":"1.0","event_id":"sha256:bc5ebef07fc3eadb629fb191efb7d208582dc5f5a6d1688f6ee0f946af7635c2"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/I44TW35NPR5PJAAZBXKBHS6KEN/bundle.json","state_url":"https://pith.science/pith/I44TW35NPR5PJAAZBXKBHS6KEN/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/I44TW35NPR5PJAAZBXKBHS6KEN/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-06-10T22:55:33Z","links":{"resolver":"https://pith.science/pith/I44TW35NPR5PJAAZBXKBHS6KEN","bundle":"https://pith.science/pith/I44TW35NPR5PJAAZBXKBHS6KEN/bundle.json","state":"https://pith.science/pith/I44TW35NPR5PJAAZBXKBHS6KEN/state.json","well_known_bundle":"https://pith.science/.well-known/pith/I44TW35NPR5PJAAZBXKBHS6KEN/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:I44TW35NPR5PJAAZBXKBHS6KEN","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"34c28eddc3a84ec52591111031a61ace34d1728a5b1e5a47f6dd1f16ec7690d0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-11-30T14:54:12Z","title_canon_sha256":"58a3a4d390f8261f81cba59cb1c194c6484e3151b0a82d6103dee92fb921eba3"},"schema_version":"1.0","source":{"id":"2512.00920","kind":"arxiv","version":5}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2512.00920","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"arxiv_version","alias_value":"2512.00920v5","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2512.00920","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"pith_short_12","alias_value":"I44TW35NPR5P","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"pith_short_16","alias_value":"I44TW35NPR5PJAAZ","created_at":"2026-05-20T00:00:29Z"},{"alias_kind":"pith_short_8","alias_value":"I44TW35N","created_at":"2026-05-20T00:00:29Z"}],"graph_snapshots":[{"event_id":"sha256:bc5ebef07fc3eadb629fb191efb7d208582dc5f5a6d1688f6ee0f946af7635c2","target":"graph","created_at":"2026-05-20T00:00:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":4,"items":[{"attestation":"unclaimed","claim_id":"C1","kind":"strongest_claim","source":"verdict.strongest_claim","status":"machine_extracted","text":"Under real-world perturbed scenarios, Reward Auditor quantifies statistical significance and effect size by auditing distribution degradation of RM preference perception confidence. This enables inference of both the certainty and severity of RM vulnerabilities across diverse real-world scenarios."},{"attestation":"unclaimed","claim_id":"C2","kind":"weakest_assumption","source":"verdict.weakest_assumption","status":"machine_extracted","text":"The chosen real-world perturbations and the definition of suitability as conditional reliability under those perturbations accurately capture the vulnerabilities that matter for safe LLM alignment in deployment."},{"attestation":"unclaimed","claim_id":"C3","kind":"one_line_summary","source":"verdict.one_line_summary","status":"machine_extracted","text":"Reward Auditor is a statistical auditing framework that infers systematic vulnerabilities in reward models by quantifying distribution degradation of preference perception confidence under real-world perturbations."},{"attestation":"unclaimed","claim_id":"C4","kind":"headline","source":"verdict.pith_extraction.headline","status":"machine_extracted","text":"Reward Auditor uses hypothesis testing to detect if reward models have systematic vulnerabilities under real-world perturbations."}],"snapshot_sha256":"eafe369da999e7fbc1213aa5908723cf0856322c49bd8e9a0638dde7e93c3b8f"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2512.00920/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Reliable reward models (RMs) are critical for ensuring the safe alignment of large language models (LLMs). However, current RM evaluation methods focus solely on preference perception accuracies in given specific scenarios, obscuring the critical vulnerabilities of RMs in real-world scenarios. We identify the true challenge lies in assessing a novel dimension: Suitability, defined as conditional reliability under specific real-world perturbations. To this end, we introduce Reward Auditor, a hypothesis-testing framework specifically designed for RM suitability inference. Rather than answering \"","authors_text":"Binhong Li, Hui Liu, Jianxiang Zang, Nijia Mo, Qiang Sun, Ruxue Bai, Shiyu Jiang, Yongda Wei","cross_cats":[],"headline":"Reward Auditor uses hypothesis testing to detect if reward models have systematic vulnerabilities under real-world perturbations.","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-11-30T14:54:12Z","title":"Reward Auditor: Inference on Reward Modeling Suitability in Real-World Perturbed Scenarios"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2512.00920","kind":"arxiv","version":5},"verdict":{"created_at":"2026-05-17T02:53:18.202770Z","id":"998e36d9-10c0-4cf3-8cf6-b8ed7d67288d","model_set":{"reader":"grok-4.3"},"one_line_summary":"Reward Auditor is a statistical auditing framework that infers systematic vulnerabilities in reward models by quantifying distribution degradation of preference perception confidence under real-world perturbations.","pipeline_version":"pith-pipeline@v0.9.0","pith_extraction_headline":"Reward Auditor uses hypothesis testing to detect if reward models have systematic vulnerabilities under real-world perturbations.","strongest_claim":"Under real-world perturbed scenarios, Reward Auditor quantifies statistical significance and effect size by auditing distribution degradation of RM preference perception confidence. This enables inference of both the certainty and severity of RM vulnerabilities across diverse real-world scenarios.","weakest_assumption":"The chosen real-world perturbations and the definition of suitability as conditional reliability under those perturbations accurately capture the vulnerabilities that matter for safe LLM alignment in deployment."}},"verdict_id":"998e36d9-10c0-4cf3-8cf6-b8ed7d67288d"}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:464e48d6fdf6c3788a826658fc652f2c1bd2f6b0dcd42b6b969d2bf403515e31","target":"record","created_at":"2026-05-20T00:00:29Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"34c28eddc3a84ec52591111031a61ace34d1728a5b1e5a47f6dd1f16ec7690d0","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-11-30T14:54:12Z","title_canon_sha256":"58a3a4d390f8261f81cba59cb1c194c6484e3151b0a82d6103dee92fb921eba3"},"schema_version":"1.0","source":{"id":"2512.00920","kind":"arxiv","version":5}},"canonical_sha256":"47393b6fad7c7af480190dd413cbca234016c43855b6f69389c34c9d00954930","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"47393b6fad7c7af480190dd413cbca234016c43855b6f69389c34c9d00954930","first_computed_at":"2026-05-20T00:00:29.144435Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-05-20T00:00:29.144435Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"SpbvHbNk5O/K6MNokjzhNnK92konFYZcgUPr2Z9z/XpBWVnC9ZsVUo7GZyi5JB/brAOdxv2EnkzqTcIAwm7hDw==","signature_status":"signed_v1","signed_at":"2026-05-20T00:00:29.145123Z","signed_message":"canonical_sha256_bytes"},"source_id":"2512.00920","source_kind":"arxiv","source_version":5}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:464e48d6fdf6c3788a826658fc652f2c1bd2f6b0dcd42b6b969d2bf403515e31","sha256:bc5ebef07fc3eadb629fb191efb7d208582dc5f5a6d1688f6ee0f946af7635c2"],"state_sha256":"0d8099ed49e710a702322a91f638f4792907439c5e571af46e7bc612568e8d34"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"S53+ycv09POmPeAxyTKpJ/emF0MdUXeXcdQTYNL9X8An6eKTX5C6CRPQZw4R5Ur/4eByEJKxi65/W1tOFZOMCw==","signed_message":"bundle_sha256_bytes","signed_at":"2026-06-10T22:55:33.538877Z","bundle_sha256":"10659ec3f19865b57e39bedf83612f9e37ff1d5bba638d18ea3f78e65e3613c7"}}