{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2025:BCCG33WCZ7K5ZLRHQ6ISBZFQYK","short_pith_number":"pith:BCCG33WC","canonical_record":{"source":{"id":"2505.20505","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T20:15:00Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"058f41e743265e6d374027b60a0df9a326bb7df02147d050272af0209616361f","abstract_canon_sha256":"78f592552bca73016f29ed79e8fe773f5fdaeab5d217ca62bc2960d52a7e5791"},"schema_version":"1.0"},"canonical_sha256":"08846deec2cfd5dcae27879120e4b0c2bdf82bed24e5495a11b1fac4b359262d","source":{"kind":"arxiv","id":"2505.20505","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.20505","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"arxiv_version","alias_value":"2505.20505v1","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.20505","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"pith_short_12","alias_value":"BCCG33WCZ7K5","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"pith_short_16","alias_value":"BCCG33WCZ7K5ZLRH","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"pith_short_8","alias_value":"BCCG33WC","created_at":"2026-07-05T11:10:13Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2025:BCCG33WCZ7K5ZLRHQ6ISBZFQYK","target":"record","payload":{"canonical_record":{"source":{"id":"2505.20505","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T20:15:00Z","cross_cats_sorted":["cs.SE"],"title_canon_sha256":"058f41e743265e6d374027b60a0df9a326bb7df02147d050272af0209616361f","abstract_canon_sha256":"78f592552bca73016f29ed79e8fe773f5fdaeab5d217ca62bc2960d52a7e5791"},"schema_version":"1.0"},"canonical_sha256":"08846deec2cfd5dcae27879120e4b0c2bdf82bed24e5495a11b1fac4b359262d","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:10:13.751605Z","signature_b64":"Q0iZMiUjr34/oxZuwcz/ql0D47rLOZbSUbN1bB++pKIw+s2fCD9Ip4RHeqAIFYALCoV/RWQvDyLrgOXr3EvXBw==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"08846deec2cfd5dcae27879120e4b0c2bdf82bed24e5495a11b1fac4b359262d","last_reissued_at":"2026-07-05T11:10:13.751064Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:10:13.751064Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2505.20505","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:10:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"HZSnP8G3pLIzfHmGfhk+1NHinzuHmh7P7lkcCBGJPB5DacDLlzsidXxiJ7UuBtSXDzKeTKyJw4IhgT3tNYbgBw==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T07:40:15.190472Z"},"content_sha256":"6e79594a4ad832656dc005bfa6208161522f14f8895dcd3f59a7a219389c154f","schema_version":"1.0","event_id":"sha256:6e79594a4ad832656dc005bfa6208161522f14f8895dcd3f59a7a219389c154f"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2025:BCCG33WCZ7K5ZLRHQ6ISBZFQYK","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"Large Language Models for IT Automation Tasks: Are We There Yet?","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["cs.SE"],"primary_cat":"cs.CL","authors_text":"Akond Rahman, John Salvador, Md Mahadi Hassan, Santu Karmaker","submitted_at":"2025-05-26T20:15:00Z","abstract_excerpt":"LLMs show promise in code generation, yet their effectiveness for IT automation tasks, particularly for tools like Ansible, remains understudied. Existing benchmarks rely primarily on synthetic tasks that fail to capture the needs of practitioners who use IT automation tools, such as Ansible. We present ITAB (IT Automation Task Benchmark), a benchmark of 126 diverse tasks (e.g., configuring servers, managing files) where each task accounts for state reconciliation: a property unique to IT automation tools. ITAB evaluates LLMs' ability to generate functional Ansible automation scripts via dynam"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.20505","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2505.20505/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T11:10:13Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"RaztysH89RLXRstb42o2adhBZawWty3bXq+0MWm+T10tclE8Pzs15qnTR9FWyVWXvvuiAhmimuuMo2uzentcCA==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-11T07:40:15.190800Z"},"content_sha256":"0fa649759e6b5f88ea41309dc251cd5c6f73dcefe01c8c6bcaac3e32054e233f","schema_version":"1.0","event_id":"sha256:0fa649759e6b5f88ea41309dc251cd5c6f73dcefe01c8c6bcaac3e32054e233f"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BCCG33WCZ7K5ZLRHQ6ISBZFQYK/bundle.json","state_url":"https://pith.science/pith/BCCG33WCZ7K5ZLRHQ6ISBZFQYK/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BCCG33WCZ7K5ZLRHQ6ISBZFQYK/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-11T07:40:15Z","links":{"resolver":"https://pith.science/pith/BCCG33WCZ7K5ZLRHQ6ISBZFQYK","bundle":"https://pith.science/pith/BCCG33WCZ7K5ZLRHQ6ISBZFQYK/bundle.json","state":"https://pith.science/pith/BCCG33WCZ7K5ZLRHQ6ISBZFQYK/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BCCG33WCZ7K5ZLRHQ6ISBZFQYK/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2025:BCCG33WCZ7K5ZLRHQ6ISBZFQYK","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"78f592552bca73016f29ed79e8fe773f5fdaeab5d217ca62bc2960d52a7e5791","cross_cats_sorted":["cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T20:15:00Z","title_canon_sha256":"058f41e743265e6d374027b60a0df9a326bb7df02147d050272af0209616361f"},"schema_version":"1.0","source":{"id":"2505.20505","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2505.20505","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"arxiv_version","alias_value":"2505.20505v1","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2505.20505","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"pith_short_12","alias_value":"BCCG33WCZ7K5","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"pith_short_16","alias_value":"BCCG33WCZ7K5ZLRH","created_at":"2026-07-05T11:10:13Z"},{"alias_kind":"pith_short_8","alias_value":"BCCG33WC","created_at":"2026-07-05T11:10:13Z"}],"graph_snapshots":[{"event_id":"sha256:0fa649759e6b5f88ea41309dc251cd5c6f73dcefe01c8c6bcaac3e32054e233f","target":"graph","created_at":"2026-07-05T11:10:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2505.20505/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"LLMs show promise in code generation, yet their effectiveness for IT automation tasks, particularly for tools like Ansible, remains understudied. Existing benchmarks rely primarily on synthetic tasks that fail to capture the needs of practitioners who use IT automation tools, such as Ansible. We present ITAB (IT Automation Task Benchmark), a benchmark of 126 diverse tasks (e.g., configuring servers, managing files) where each task accounts for state reconciliation: a property unique to IT automation tools. ITAB evaluates LLMs' ability to generate functional Ansible automation scripts via dynam","authors_text":"Akond Rahman, John Salvador, Md Mahadi Hassan, Santu Karmaker","cross_cats":["cs.SE"],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T20:15:00Z","title":"Large Language Models for IT Automation Tasks: Are We There Yet?"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2505.20505","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:6e79594a4ad832656dc005bfa6208161522f14f8895dcd3f59a7a219389c154f","target":"record","created_at":"2026-07-05T11:10:13Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"78f592552bca73016f29ed79e8fe773f5fdaeab5d217ca62bc2960d52a7e5791","cross_cats_sorted":["cs.SE"],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-05-26T20:15:00Z","title_canon_sha256":"058f41e743265e6d374027b60a0df9a326bb7df02147d050272af0209616361f"},"schema_version":"1.0","source":{"id":"2505.20505","kind":"arxiv","version":1}},"canonical_sha256":"08846deec2cfd5dcae27879120e4b0c2bdf82bed24e5495a11b1fac4b359262d","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"08846deec2cfd5dcae27879120e4b0c2bdf82bed24e5495a11b1fac4b359262d","first_computed_at":"2026-07-05T11:10:13.751064Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T11:10:13.751064Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"Q0iZMiUjr34/oxZuwcz/ql0D47rLOZbSUbN1bB++pKIw+s2fCD9Ip4RHeqAIFYALCoV/RWQvDyLrgOXr3EvXBw==","signature_status":"signed_v1","signed_at":"2026-07-05T11:10:13.751605Z","signed_message":"canonical_sha256_bytes"},"source_id":"2505.20505","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:6e79594a4ad832656dc005bfa6208161522f14f8895dcd3f59a7a219389c154f","sha256:0fa649759e6b5f88ea41309dc251cd5c6f73dcefe01c8c6bcaac3e32054e233f"],"state_sha256":"978ad77f03d127cdb22a6ccb34b989c99930bac94ea035b2e2d6970077b3e058"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"2wvB037bxXUHDkgSSB7D1v9EgOADTccN4ryzQV2e9uUjrreHLeENSSMSH3of3/lsQwGjlQWA0qF0V7lB36GNDg==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-11T07:40:15.193084Z","bundle_sha256":"3254b41aabdb3cc79bf746f9f7811003a965a19b8266fc0a85561527a7969e86"}}