{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:K7UM5ZW7KDHSOQZQPF5R3ZHACS","short_pith_number":"pith:K7UM5ZW7","canonical_record":{"source":{"id":"2503.09347","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T12:49:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"024806904100a7485973da9d3879f2f4d585f8a196f1350122952b6f5360ae8c","abstract_canon_sha256":"806a5836d5c1174f815d4f570bc9e5204d21e74c2f52741f93516b818b3e4e21"},"schema_version":"1.0"},"canonical_sha256":"57e8cee6df50cf274330797b1de4e01498efd27ca84ccf088bc3f44c4ded8dc0","source":{"kind":"arxiv","id":"2503.09347","version":3},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.09347","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"arxiv_version","alias_value":"2503.09347v3","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.09347","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"pith_short_12","alias_value":"K7UM5ZW7KDHS","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"pith_short_16","alias_value":"K7UM5ZW7KDHSOQZQ","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"pith_short_8","alias_value":"K7UM5ZW7","created_at":"2026-07-05T11:34:05Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:K7UM5ZW7KDHSOQZQPF5R3ZHACS","target":"record","payload":{"canonical_record":{"source":{"id":"2503.09347","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T12:49:02Z","cross_cats_sorted":["cs.AI"],"title_canon_sha256":"024806904100a7485973da9d3879f2f4d585f8a196f1350122952b6f5360ae8c","abstract_canon_sha256":"806a5836d5c1174f815d4f570bc9e5204d21e74c2f52741f93516b818b3e4e21"},"schema_version":"1.0"},"canonical_sha256":"57e8cee6df50cf274330797b1de4e01498efd27ca84ccf088bc3f44c4ded8dc0","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:34:05.732169Z","signature_b64":"xfyuw3QIATw9bclaIIayxLby/HbtUtjjvESz8nKYNo3MhjcD6CS6MjBa6jVv8tTv69nPUlcbQIp5P23c0U/OAQ==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"57e8cee6df50cf274330797b1de4e01498efd27ca84ccf088bc3f44c4ded8dc0","last_reissued_at":"2026-07-05T11:34:05.731663Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:34:05.731663Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2503.09347","source_version":3,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:34:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"aIUdTi5F5LSC5213jefLD8+Lwk0O/fi5pP5kkOOKWZfuEvnaby4jJcPJN9HW2ggquMdUqYfbnQEXy1bAQ8iqBg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T12:47:14.817701Z"},"content_sha256":"f70a7372f718975bf9f4df144eb72b3ff54d2e9193c7fb6412eaa4be5ab190b2","schema_version":"1.0","event_id":"sha256:f70a7372f718975bf9f4df144eb72b3ff54d2e9193c7fb6412eaa4be5ab190b2"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:K7UM5ZW7KDHSOQZQPF5R3ZHACS","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Safer or Luckier? LLMs as Safety Evaluators Are Not Robust to Artifacts","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI"],"primary_cat":"cs.CL","authors_text":"Hongyu Chen, Seraphina Goldfarb-Tarrant","submitted_at":"2025-03-12T12:49:02Z","abstract_excerpt":"Large Language Models (LLMs) are increasingly employed as automated evaluators to assess the safety of generated content, yet their reliability in this role remains uncertain. This study evaluates a diverse set of 11 LLM judge models across critical safety domains, examining three key aspects: self-consistency in repeated judging tasks, alignment with human judgments, and susceptibility to input artifacts such as apologetic or verbose phrasing. Our findings reveal that biases in LLM judges can significantly distort the final verdict on which content source is safer, undermining the validity of"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.09347","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2503.09347/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:34:05Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"KxyUqc4liS8xbtLMI4I4gynsH1RfR2dW+WzoCV0VsCNQWBNwu20ghWhnsNhyN8OirHipzgY1kZzfLn63slmDBA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-10T12:47:14.818395Z"},"content_sha256":"c64bd6ce57c4915a4daef944bd60622cf3cda129af2bc00dcdae8df1422d7a31","schema_version":"1.0","event_id":"sha256:c64bd6ce57c4915a4daef944bd60622cf3cda129af2bc00dcdae8df1422d7a31"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/K7UM5ZW7KDHSOQZQPF5R3ZHACS/bundle.json","state_url":"https://pith.science/pith/K7UM5ZW7KDHSOQZQPF5R3ZHACS/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/K7UM5ZW7KDHSOQZQPF5R3ZHACS/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-10T12:47:14Z","links":{"resolver":"https://pith.science/pith/K7UM5ZW7KDHSOQZQPF5R3ZHACS","bundle":"https://pith.science/pith/K7UM5ZW7KDHSOQZQPF5R3ZHACS/bundle.json","state":"https://pith.science/pith/K7UM5ZW7KDHSOQZQPF5R3ZHACS/state.json","well_known_bundle":"https://pith.science/.well-known/pith/K7UM5ZW7KDHSOQZQPF5R3ZHACS/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:K7UM5ZW7KDHSOQZQPF5R3ZHACS","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"806a5836d5c1174f815d4f570bc9e5204d21e74c2f52741f93516b818b3e4e21","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T12:49:02Z","title_canon_sha256":"024806904100a7485973da9d3879f2f4d585f8a196f1350122952b6f5360ae8c"},"schema_version":"1.0","source":{"id":"2503.09347","kind":"arxiv","version":3}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2503.09347","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"arxiv_version","alias_value":"2503.09347v3","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2503.09347","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"pith_short_12","alias_value":"K7UM5ZW7KDHS","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"pith_short_16","alias_value":"K7UM5ZW7KDHSOQZQ","created_at":"2026-07-05T11:34:05Z"},{"alias_kind":"pith_short_8","alias_value":"K7UM5ZW7","created_at":"2026-07-05T11:34:05Z"}],"graph_snapshots":[{"event_id":"sha256:c64bd6ce57c4915a4daef944bd60622cf3cda129af2bc00dcdae8df1422d7a31","target":"graph","created_at":"2026-07-05T11:34:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2503.09347/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Large Language Models (LLMs) are increasingly employed as automated evaluators to assess the safety of generated content, yet their reliability in this role remains uncertain. This study evaluates a diverse set of 11 LLM judge models across critical safety domains, examining three key aspects: self-consistency in repeated judging tasks, alignment with human judgments, and susceptibility to input artifacts such as apologetic or verbose phrasing. Our findings reveal that biases in LLM judges can significantly distort the final verdict on which content source is safer, undermining the validity of","authors_text":"Hongyu Chen, Seraphina Goldfarb-Tarrant","cross_cats":["cs.AI"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T12:49:02Z","title":"Safer or Luckier? LLMs as Safety Evaluators Are Not Robust to Artifacts"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2503.09347","kind":"arxiv","version":3},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:f70a7372f718975bf9f4df144eb72b3ff54d2e9193c7fb6412eaa4be5ab190b2","target":"record","created_at":"2026-07-05T11:34:05Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"806a5836d5c1174f815d4f570bc9e5204d21e74c2f52741f93516b818b3e4e21","cross_cats_sorted":["cs.AI"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-03-12T12:49:02Z","title_canon_sha256":"024806904100a7485973da9d3879f2f4d585f8a196f1350122952b6f5360ae8c"},"schema_version":"1.0","source":{"id":"2503.09347","kind":"arxiv","version":3}},"canonical_sha256":"57e8cee6df50cf274330797b1de4e01498efd27ca84ccf088bc3f44c4ded8dc0","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"57e8cee6df50cf274330797b1de4e01498efd27ca84ccf088bc3f44c4ded8dc0","first_computed_at":"2026-07-05T11:34:05.731663Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:34:05.731663Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"xfyuw3QIATw9bclaIIayxLby/HbtUtjjvESz8nKYNo3MhjcD6CS6MjBa6jVv8tTv69nPUlcbQIp5P23c0U/OAQ==","signature_status":"signed_v1","signed_at":"2026-07-05T11:34:05.732169Z","signed_message":"canonical_sha256_bytes"},"source_id":"2503.09347","source_kind":"arxiv","source_version":3}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:f70a7372f718975bf9f4df144eb72b3ff54d2e9193c7fb6412eaa4be5ab190b2","sha256:c64bd6ce57c4915a4daef944bd60622cf3cda129af2bc00dcdae8df1422d7a31"],"state_sha256":"7bd2475f80ef5cdcdae008c8652632b6cb3bf7a0d5e217b794eb0aef41ba5a0c"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"25c6C8vLJI/q96CgiaeMOqPWWlHQ2dXPHYXoqweuAFCx9N/BBZrlzgYrTxuRh4hM6Tma439FRWsX4WCQfo00DA==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-10T12:47:14.823439Z","bundle_sha256":"2a216a583c68ecc352a9fb31fec3fd521182561d136828fe6c47f4d265c9964c"}}