{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2023:B3X2PZ7B7KT6SQPQEAOCVE3FTC","short_pith_number":"pith:B3X2PZ7B","canonical_record":{"source":{"id":"2307.10558","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-07-20T03:54:24Z","cross_cats_sorted":[],"title_canon_sha256":"bc3326c5b3e819f8d727b87d4d75edb372f681432bcc0abdb069752b3891ef9f","abstract_canon_sha256":"fcf5873452d3db15d50b95db71ce31b9e5b529480c784667068f70d6313cd332"},"schema_version":"1.0"},"canonical_sha256":"0eefa7e7e1faa7e941f0201c2a936598b71fd63b8a802e11e7bec66c494926db","source":{"kind":"arxiv","id":"2307.10558","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2307.10558","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"arxiv_version","alias_value":"2307.10558v2","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2307.10558","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"pith_short_12","alias_value":"B3X2PZ7B7KT6","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"pith_short_16","alias_value":"B3X2PZ7B7KT6SQPQ","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"pith_short_8","alias_value":"B3X2PZ7B","created_at":"2026-07-05T08:03:06Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2023:B3X2PZ7B7KT6SQPQEAOCVE3FTC","target":"record","payload":{"canonical_record":{"source":{"id":"2307.10558","kind":"arxiv","version":2},"metadata":{"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-07-20T03:54:24Z","cross_cats_sorted":[],"title_canon_sha256":"bc3326c5b3e819f8d727b87d4d75edb372f681432bcc0abdb069752b3891ef9f","abstract_canon_sha256":"fcf5873452d3db15d50b95db71ce31b9e5b529480c784667068f70d6313cd332"},"schema_version":"1.0"},"canonical_sha256":"0eefa7e7e1faa7e941f0201c2a936598b71fd63b8a802e11e7bec66c494926db","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:03:06.482463Z","signature_b64":"JpO1nsHOlPtayLu3RRBFfs26ipmavpRxzTBRdybPA8nhVZpvziT+0RiYi/nW6LMNsSP5ZcHcd1LDPmIkAapiDw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0eefa7e7e1faa7e941f0201c2a936598b71fd63b8a802e11e7bec66c494926db","last_reissued_at":"2026-07-05T08:03:06.481993Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:03:06.481993Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2307.10558","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:03:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"yScyhe/Vq+9ZSvz06DZoZySVHFnGVI7NKkimy/HIVT1WHWINZ1D4dHt/Q9s7on4sJ+EmQa/WYykcbv6ndWoACQ==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T06:33:20.540754Z"},"content_sha256":"bac9f6ee37af093c00f791932a726551025123c0e55c1f5c9902502d9597d424","schema_version":"1.0","event_id":"sha256:bac9f6ee37af093c00f791932a726551025123c0e55c1f5c9902502d9597d424"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2023:B3X2PZ7B7KT6SQPQEAOCVE3FTC","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Instruction-following Evaluation through Verbalizer Manipulation","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Hai Wang, Hongxia Jin, Jun Yan, Shiyang Li, Vijay Srinivasan, Xiang Ren, Zheng Tang","submitted_at":"2023-07-20T03:54:24Z","abstract_excerpt":"While instruction-tuned models have shown remarkable success in various natural language processing tasks, accurately evaluating their ability to follow instructions remains challenging. Existing benchmarks primarily focus on common instructions that align well with what the model learned during training. However, proficiency in responding to these instructions does not necessarily imply strong ability in instruction following. In this paper, we propose a novel instruction-following evaluation protocol called verbalizer manipulation. It instructs the model to verbalize the task label with word"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2307.10558","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2307.10558/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:03:06Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DM1XnyzfT4OC//DImDgQQVsmYjgS1kQqsI3d7/tKJe7+MEU43HhgOKl+zlxsfOwovgeKa4HjMvpLlGtFBAb2Bg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-07-07T06:33:20.541445Z"},"content_sha256":"236cd7a62d304ae284b1ea9da19c3a5cb2e57fd2df24677acadc110a92e2e870","schema_version":"1.0","event_id":"sha256:236cd7a62d304ae284b1ea9da19c3a5cb2e57fd2df24677acadc110a92e2e870"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/B3X2PZ7B7KT6SQPQEAOCVE3FTC/bundle.json","state_url":"https://pith.science/pith/B3X2PZ7B7KT6SQPQEAOCVE3FTC/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/B3X2PZ7B7KT6SQPQEAOCVE3FTC/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-07-07T06:33:20Z","links":{"resolver":"https://pith.science/pith/B3X2PZ7B7KT6SQPQEAOCVE3FTC","bundle":"https://pith.science/pith/B3X2PZ7B7KT6SQPQEAOCVE3FTC/bundle.json","state":"https://pith.science/pith/B3X2PZ7B7KT6SQPQEAOCVE3FTC/state.json","well_known_bundle":"https://pith.science/.well-known/pith/B3X2PZ7B7KT6SQPQEAOCVE3FTC/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2023:B3X2PZ7B7KT6SQPQEAOCVE3FTC","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"fcf5873452d3db15d50b95db71ce31b9e5b529480c784667068f70d6313cd332","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-07-20T03:54:24Z","title_canon_sha256":"bc3326c5b3e819f8d727b87d4d75edb372f681432bcc0abdb069752b3891ef9f"},"schema_version":"1.0","source":{"id":"2307.10558","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2307.10558","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"arxiv_version","alias_value":"2307.10558v2","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2307.10558","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"pith_short_12","alias_value":"B3X2PZ7B7KT6","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"pith_short_16","alias_value":"B3X2PZ7B7KT6SQPQ","created_at":"2026-07-05T08:03:06Z"},{"alias_kind":"pith_short_8","alias_value":"B3X2PZ7B","created_at":"2026-07-05T08:03:06Z"}],"graph_snapshots":[{"event_id":"sha256:236cd7a62d304ae284b1ea9da19c3a5cb2e57fd2df24677acadc110a92e2e870","target":"graph","created_at":"2026-07-05T08:03:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2307.10558/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"While instruction-tuned models have shown remarkable success in various natural language processing tasks, accurately evaluating their ability to follow instructions remains challenging. Existing benchmarks primarily focus on common instructions that align well with what the model learned during training. However, proficiency in responding to these instructions does not necessarily imply strong ability in instruction following. In this paper, we propose a novel instruction-following evaluation protocol called verbalizer manipulation. It instructs the model to verbalize the task label with word","authors_text":"Hai Wang, Hongxia Jin, Jun Yan, Shiyang Li, Vijay Srinivasan, Xiang Ren, Zheng Tang","cross_cats":[],"headline":"","license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-07-20T03:54:24Z","title":"Instruction-following Evaluation through Verbalizer Manipulation"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2307.10558","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:bac9f6ee37af093c00f791932a726551025123c0e55c1f5c9902502d9597d424","target":"record","created_at":"2026-07-05T08:03:06Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"fcf5873452d3db15d50b95db71ce31b9e5b529480c784667068f70d6313cd332","cross_cats_sorted":[],"license":"http://arxiv.org/licenses/nonexclusive-distrib/1.0/","primary_cat":"cs.CL","submitted_at":"2023-07-20T03:54:24Z","title_canon_sha256":"bc3326c5b3e819f8d727b87d4d75edb372f681432bcc0abdb069752b3891ef9f"},"schema_version":"1.0","source":{"id":"2307.10558","kind":"arxiv","version":2}},"canonical_sha256":"0eefa7e7e1faa7e941f0201c2a936598b71fd63b8a802e11e7bec66c494926db","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0eefa7e7e1faa7e941f0201c2a936598b71fd63b8a802e11e7bec66c494926db","first_computed_at":"2026-07-05T08:03:06.481993Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:03:06.481993Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"JpO1nsHOlPtayLu3RRBFfs26ipmavpRxzTBRdybPA8nhVZpvziT+0RiYi/nW6LMNsSP5ZcHcd1LDPmIkAapiDw==","signature_status":"signed_v1","signed_at":"2026-07-05T08:03:06.482463Z","signed_message":"canonical_sha256_bytes"},"source_id":"2307.10558","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:bac9f6ee37af093c00f791932a726551025123c0e55c1f5c9902502d9597d424","sha256:236cd7a62d304ae284b1ea9da19c3a5cb2e57fd2df24677acadc110a92e2e870"],"state_sha256":"7bfc4bd881949bcb8d90a9530a45d1342bcc03da72fe266c1ee09341fb40780e"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Dl3ftzQ9x5mfEivy03bq9wSLYJnypIXEVBU3NcUGbzicXq1GrXUTWGucC4HMs4C5aXGE+V6rwEFCSdGiGPlSCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-07-07T06:33:20.545039Z","bundle_sha256":"8af70e9a573f230525b10691ed053f715a87025781658e9347cb61b006894eb4"}}