{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:4SNZL2V46ZHOESRZIPYAIIIF44","short_pith_number":"pith:4SNZL2V4","canonical_record":{"source":{"id":"2406.16714","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-24T15:16:45Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"a99e15206330138a053e890154ded3ebfa66f0aa60ca60e9914c9153350a035c","abstract_canon_sha256":"275a1a94704e581dba582a4e437f3cdb571fd8531476a55a9599fcc41c621801"},"schema_version":"1.0"},"canonical_sha256":"e49b95eabcf64ee24a3943f0042105e70b807d060b1d80fc7f1b5406b4d8fdc1","source":{"kind":"arxiv","id":"2406.16714","version":2},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.16714","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"arxiv_version","alias_value":"2406.16714v2","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.16714","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"pith_short_12","alias_value":"4SNZL2V46ZHO","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"pith_short_16","alias_value":"4SNZL2V46ZHOESRZ","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"pith_short_8","alias_value":"4SNZL2V4","created_at":"2026-07-05T09:46:48Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:4SNZL2V46ZHOESRZIPYAIIIF44","target":"record","payload":{"canonical_record":{"source":{"id":"2406.16714","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-24T15:16:45Z","cross_cats_sorted":["cs.AI","cs.LG"],"title_canon_sha256":"a99e15206330138a053e890154ded3ebfa66f0aa60ca60e9914c9153350a035c","abstract_canon_sha256":"275a1a94704e581dba582a4e437f3cdb571fd8531476a55a9599fcc41c621801"},"schema_version":"1.0"},"canonical_sha256":"e49b95eabcf64ee24a3943f0042105e70b807d060b1d80fc7f1b5406b4d8fdc1","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T09:46:48.318963Z","signature_b64":"AsCy9/XMRwXRClnv+hCQ6fj/5DI0nSRrQzY66JjD6fwJfQTeq3lVUzc5RUzDHZyWgkcRpr5Zgf2oL9XOUZHYDg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"e49b95eabcf64ee24a3943f0042105e70b807d060b1d80fc7f1b5406b4d8fdc1","last_reissued_at":"2026-07-05T09:46:48.318374Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T09:46:48.318374Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2406.16714","source_version":2,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:46:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"T2er7ZwNalq2ZdHMd8iicbuRHZaRPcq11A1bNoIqfdt98LlmOrjfz1Lj+6XuyHbvlWhPQEJPzRmPDlIHemIFAw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T01:42:57.610044Z"},"content_sha256":"6bc97eeabcec1cb98737f08449cfc39611bedfd24e96d29440e140f54b6010f8","schema_version":"1.0","event_id":"sha256:6bc97eeabcec1cb98737f08449cfc39611bedfd24e96d29440e140f54b6010f8"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:4SNZL2V46ZHOESRZIPYAIIIF44","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"AutoDetect: Towards a Unified Framework for Automated Weakness Detection in Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.AI","cs.LG"],"primary_cat":"cs.CL","authors_text":"Hongning Wang, Jiale Cheng, Jie Tang, Minlie Huang, Pei Ke, Xiao Liu, Xiaotao Gu, Yida Lu, Yuxiao Dong","submitted_at":"2024-06-24T15:16:45Z","abstract_excerpt":"Although Large Language Models (LLMs) are becoming increasingly powerful, they still exhibit significant but subtle weaknesses, such as mistakes in instruction-following or coding tasks. As these unexpected errors could lead to severe consequences in practical deployments, it is crucial to investigate the limitations within LLMs systematically. Traditional benchmarking approaches cannot thoroughly pinpoint specific model deficiencies, while manual inspections are costly and not scalable. In this paper, we introduce a unified framework, AutoDetect, to automatically expose weaknesses in LLMs acr"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.16714","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2406.16714/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T09:46:48Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"ko9K8Qf2cAV9wnmjhnlf8Qps8P/NW7lAWJgZJxnZ4tVRuEXWDW90FdYi7J7+0r/UMUiRbm5Co8uho8nRmYz9Dw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-15T01:42:57.610595Z"},"content_sha256":"28080de78cc37f634de7de5c83c7eab9fadaa01b4b78f47d2e87186d9c445778","schema_version":"1.0","event_id":"sha256:28080de78cc37f634de7de5c83c7eab9fadaa01b4b78f47d2e87186d9c445778"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/4SNZL2V46ZHOESRZIPYAIIIF44/bundle.json","state_url":"https://pith.science/pith/4SNZL2V46ZHOESRZIPYAIIIF44/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/4SNZL2V46ZHOESRZIPYAIIIF44/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-15T01:42:57Z","links":{"resolver":"https://pith.science/pith/4SNZL2V46ZHOESRZIPYAIIIF44","bundle":"https://pith.science/pith/4SNZL2V46ZHOESRZIPYAIIIF44/bundle.json","state":"https://pith.science/pith/4SNZL2V46ZHOESRZIPYAIIIF44/state.json","well_known_bundle":"https://pith.science/.well-known/pith/4SNZL2V46ZHOESRZIPYAIIIF44/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:4SNZL2V46ZHOESRZIPYAIIIF44","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"275a1a94704e581dba582a4e437f3cdb571fd8531476a55a9599fcc41c621801","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-24T15:16:45Z","title_canon_sha256":"a99e15206330138a053e890154ded3ebfa66f0aa60ca60e9914c9153350a035c"},"schema_version":"1.0","source":{"id":"2406.16714","kind":"arxiv","version":2}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2406.16714","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"arxiv_version","alias_value":"2406.16714v2","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2406.16714","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"pith_short_12","alias_value":"4SNZL2V46ZHO","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"pith_short_16","alias_value":"4SNZL2V46ZHOESRZ","created_at":"2026-07-05T09:46:48Z"},{"alias_kind":"pith_short_8","alias_value":"4SNZL2V4","created_at":"2026-07-05T09:46:48Z"}],"graph_snapshots":[{"event_id":"sha256:28080de78cc37f634de7de5c83c7eab9fadaa01b4b78f47d2e87186d9c445778","target":"graph","created_at":"2026-07-05T09:46:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2406.16714/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"Although Large Language Models (LLMs) are becoming increasingly powerful, they still exhibit significant but subtle weaknesses, such as mistakes in instruction-following or coding tasks. As these unexpected errors could lead to severe consequences in practical deployments, it is crucial to investigate the limitations within LLMs systematically. Traditional benchmarking approaches cannot thoroughly pinpoint specific model deficiencies, while manual inspections are costly and not scalable. In this paper, we introduce a unified framework, AutoDetect, to automatically expose weaknesses in LLMs acr","authors_text":"Hongning Wang, Jiale Cheng, Jie Tang, Minlie Huang, Pei Ke, Xiao Liu, Xiaotao Gu, Yida Lu, Yuxiao Dong","cross_cats":["cs.AI","cs.LG"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-24T15:16:45Z","title":"AutoDetect: Towards a Unified Framework for Automated Weakness Detection in Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2406.16714","kind":"arxiv","version":2},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6bc97eeabcec1cb98737f08449cfc39611bedfd24e96d29440e140f54b6010f8","target":"record","created_at":"2026-07-05T09:46:48Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"275a1a94704e581dba582a4e437f3cdb571fd8531476a55a9599fcc41c621801","cross_cats_sorted":["cs.AI","cs.LG"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-06-24T15:16:45Z","title_canon_sha256":"a99e15206330138a053e890154ded3ebfa66f0aa60ca60e9914c9153350a035c"},"schema_version":"1.0","source":{"id":"2406.16714","kind":"arxiv","version":2}},"canonical_sha256":"e49b95eabcf64ee24a3943f0042105e70b807d060b1d80fc7f1b5406b4d8fdc1","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"e49b95eabcf64ee24a3943f0042105e70b807d060b1d80fc7f1b5406b4d8fdc1","first_computed_at":"2026-07-05T09:46:48.318374Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T09:46:48.318374Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"AsCy9/XMRwXRClnv+hCQ6fj/5DI0nSRrQzY66JjD6fwJfQTeq3lVUzc5RUzDHZyWgkcRpr5Zgf2oL9XOUZHYDg==","signature_status":"signed_v1","signed_at":"2026-07-05T09:46:48.318963Z","signed_message":"canonical_sha256_bytes"},"source_id":"2406.16714","source_kind":"arxiv","source_version":2}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6bc97eeabcec1cb98737f08449cfc39611bedfd24e96d29440e140f54b6010f8","sha256:28080de78cc37f634de7de5c83c7eab9fadaa01b4b78f47d2e87186d9c445778"],"state_sha256":"a1ed949c502f55e0f88589931c3263b46f565f295c3b9214c27ad405a8eb92a9"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"xRGyAPGEC8njNrizd8codnXR8w+LsMcJEBOYGkcJ3jbw5x6qd9cyg0ZsQ+Gvf2wGtBSBQhu88vSi6hy815PJBg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-15T01:42:57.616110Z","bundle_sha256":"a4ca292888cec7cb76a1720c36e980f0ff5e80744b9da7005ef0702150384c7f"}}