{"bundle_type":"pith_open_graph_bundle","bundle_version":"1.0","pith_number":"pith:2024:BQVZA7H5BJPDD3EWWPOPYUNZH2","short_pith_number":"pith:BQVZA7H5","canonical_record":{"source":{"id":"2408.11061","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-07T19:32:59Z","cross_cats_sorted":[],"title_canon_sha256":"140b778a599ca0e50c83e5e43475c85490a10a5a818901db2c955f2b9633cbbd","abstract_canon_sha256":"cd8b242f495f819fab4be5f3b587097eb19771be93b8f8e81e219e85345ef268"},"schema_version":"1.0"},"canonical_sha256":"0c2b907cfd0a5e31ec96b3dcfc51b93e81fbaf3d730d3bcdddd46affd0268332","source":{"kind":"arxiv","id":"2408.11061","version":1},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.11061","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"arxiv_version","alias_value":"2408.11061v1","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.11061","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"pith_short_12","alias_value":"BQVZA7H5BJPD","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"pith_short_16","alias_value":"BQVZA7H5BJPDD3EW","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"pith_short_8","alias_value":"BQVZA7H5","created_at":"2026-07-05T08:57:24Z"}],"events":[{"event_type":"record_created","subject_pith_number":"pith:2024:BQVZA7H5BJPDD3EWWPOPYUNZH2","target":"record","payload":{"canonical_record":{"source":{"id":"2408.11061","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-07T19:32:59Z","cross_cats_sorted":[],"title_canon_sha256":"140b778a599ca0e50c83e5e43475c85490a10a5a818901db2c955f2b9633cbbd","abstract_canon_sha256":"cd8b242f495f819fab4be5f3b587097eb19771be93b8f8e81e219e85345ef268"},"schema_version":"1.0"},"canonical_sha256":"0c2b907cfd0a5e31ec96b3dcfc51b93e81fbaf3d730d3bcdddd46affd0268332","receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:57:24.482675Z","signature_b64":"miB79lpct4pHmP4PRQQkZZCkt+HA5h/qkFQ0LVPw4xfJMp4pYwEGjRd84LzgtIP65TfvJ3VEIRUNoMy8kIM4Dg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0c2b907cfd0a5e31ec96b3dcfc51b93e81fbaf3d730d3bcdddd46affd0268332","last_reissued_at":"2026-07-05T08:57:24.482238Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:57:24.482238Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"source_kind":"arxiv","source_id":"2408.11061","source_version":1,"attestation_state":"computed"},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:57:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"DpgS5ZzGAQ21YHQQEA+JiXoc/piXfoEHhOeZnMClDYUIIC74819CNKMzzdpAj8oV5mctPaIwGc9W8HkFCkT3Cg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T09:46:40.651919Z"},"content_sha256":"fb009d2a93a695864033b3943f3fdfaa02763d576fe930e669236ca489f72c4e","schema_version":"1.0","event_id":"sha256:fb009d2a93a695864033b3943f3fdfaa02763d576fe930e669236ca489f72c4e"},{"event_type":"graph_snapshot","subject_pith_number":"pith:2024:BQVZA7H5BJPDD3EWWPOPYUNZH2","target":"graph","payload":{"graph_snapshot":{"paper":{"title":"StructuredRAG: JSON Response Formatting with Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Akanksha Sharma, Bob van Luijt, Charles Pierse, Connor Shorten, Erika Cardenas, John Trengrove, Thomas Benjamin Smith","submitted_at":"2024-08-07T19:32:59Z","abstract_excerpt":"The ability of Large Language Models (LLMs) to generate structured outputs, such as JSON, is crucial for their use in Compound AI Systems. However, evaluating and improving this capability remains challenging. In this work, we introduce StructuredRAG, a benchmark of six tasks designed to assess LLMs' proficiency in following response format instructions. We evaluate two state-of-the-art LLMs, Gemini 1.5 Pro and Llama 3 8B-instruct with 4-bit quantization using two distinct prompting strategies. We introduce these prompting strategies as f-String and Follow the Format (FF) prompting. Across 24 "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.11061","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.11061/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"verdict_id":null},"signer":{"signer_id":"pith.science","signer_type":"pith_registry","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"created_at":"2026-07-05T08:57:24Z","supersedes":[],"prev_event":null,"signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"NP1fvx1aXR77tEhWur3EXGz0dI6KIlmrVoi99hR7dvA3UG+eLUZK1BPNlwBU1MI/cE6U9VW6SPJiqD9KevTwDg==","signed_message":"open_graph_event_sha256_bytes","signed_at":"2026-08-09T09:46:40.652419Z"},"content_sha256":"a43a2cbacc3dee92036ab26004007fab97ad85db76463b8f242b1288131187db","schema_version":"1.0","event_id":"sha256:a43a2cbacc3dee92036ab26004007fab97ad85db76463b8f242b1288131187db"}],"timestamp_proofs":[],"mirror_hints":[{"mirror_type":"https","name":"Pith Resolver","base_url":"https://pith.science","bundle_url":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/bundle.json","state_url":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/state.json","well_known_bundle_url":"https://pith.science/.well-known/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/bundle.json","status":"primary"}],"public_keys":[{"key_id":"pith-v1-2026-05","algorithm":"ed25519","format":"raw","public_key_b64":"stVStoiQhXFxp4s2pdzPNoqVNBMojDU/fJ2db5S3CbM=","public_key_hex":"b2d552b68890857171a78b36a5dccf368a953413288c353f7c9d9d6f94b709b3","fingerprint_sha256_b32_first128bits":"RVFV5Z2OI2J3ZUO7ERDEBCYNKS","fingerprint_sha256_hex":"8d4b5ee74e4693bcd1df2446408b0d54","rotates_at":null,"url":"https://pith.science/pith-signing-key.json","notes":"Pith uses this Ed25519 key to sign canonical record SHA-256 digests. Verify with: ed25519_verify(public_key, message=canonical_sha256_bytes, signature=base64decode(signature_b64))."}],"merge_version":"pith-open-graph-merge-v1","built_at":"2026-08-09T09:46:40Z","links":{"resolver":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2","bundle":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/bundle.json","state":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/state.json","well_known_bundle":"https://pith.science/.well-known/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/bundle.json"},"state":{"state_type":"pith_open_graph_state","state_version":"1.0","pith_number":"pith:2024:BQVZA7H5BJPDD3EWWPOPYUNZH2","merge_version":"pith-open-graph-merge-v1","event_count":2,"valid_event_count":2,"invalid_event_count":0,"equivocation_count":0,"current":{"canonical_record":{"metadata":{"abstract_canon_sha256":"cd8b242f495f819fab4be5f3b587097eb19771be93b8f8e81e219e85345ef268","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-07T19:32:59Z","title_canon_sha256":"140b778a599ca0e50c83e5e43475c85490a10a5a818901db2c955f2b9633cbbd"},"schema_version":"1.0","source":{"id":"2408.11061","kind":"arxiv","version":1}},"source_aliases":[{"alias_kind":"arxiv","alias_value":"2408.11061","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"arxiv_version","alias_value":"2408.11061v1","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.11061","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"pith_short_12","alias_value":"BQVZA7H5BJPD","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"pith_short_16","alias_value":"BQVZA7H5BJPDD3EW","created_at":"2026-07-05T08:57:24Z"},{"alias_kind":"pith_short_8","alias_value":"BQVZA7H5","created_at":"2026-07-05T08:57:24Z"}],"graph_snapshots":[{"event_id":"sha256:a43a2cbacc3dee92036ab26004007fab97ad85db76463b8f242b1288131187db","target":"graph","created_at":"2026-07-05T08:57:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"graph_snapshot":{"author_claims":{"count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","strong_count":0},"builder_version":"pith-number-builder-2026-05-17-v1","claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"integrity":{"available":true,"clean":true,"detectors_run":[],"endpoint":"/pith/2408.11061/integrity.json","findings":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938","summary":{"advisory":0,"by_detector":{},"critical":0,"informational":0}},"paper":{"abstract_excerpt":"The ability of Large Language Models (LLMs) to generate structured outputs, such as JSON, is crucial for their use in Compound AI Systems. However, evaluating and improving this capability remains challenging. In this work, we introduce StructuredRAG, a benchmark of six tasks designed to assess LLMs' proficiency in following response format instructions. We evaluate two state-of-the-art LLMs, Gemini 1.5 Pro and Llama 3 8B-instruct with 4-bit quantization using two distinct prompting strategies. We introduce these prompting strategies as f-String and Follow the Format (FF) prompting. Across 24 ","authors_text":"Akanksha Sharma, Bob van Luijt, Charles Pierse, Connor Shorten, Erika Cardenas, John Trengrove, Thomas Benjamin Smith","cross_cats":[],"headline":"","license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-07T19:32:59Z","title":"StructuredRAG: JSON Response Formatting with Large Language Models"},"references":{"count":0,"internal_anchors":0,"resolved_work":0,"sample":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.11061","kind":"arxiv","version":1},"verdict":{"created_at":null,"id":null,"model_set":{},"one_line_summary":"","pipeline_version":null,"pith_extraction_headline":"","strongest_claim":"","weakest_assumption":""}},"verdict_id":null}}],"author_attestations":[],"timestamp_anchors":[],"storage_attestations":[],"citation_signatures":[],"replication_records":[],"corrections":[],"mirror_hints":[],"record_created":{"event_id":"sha256:fb009d2a93a695864033b3943f3fdfaa02763d576fe930e669236ca489f72c4e","target":"record","created_at":"2026-07-05T08:57:24Z","signer":{"key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signer_id":"pith.science","signer_type":"pith_registry"},"payload":{"attestation_state":"computed","canonical_record":{"metadata":{"abstract_canon_sha256":"cd8b242f495f819fab4be5f3b587097eb19771be93b8f8e81e219e85345ef268","cross_cats_sorted":[],"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-07T19:32:59Z","title_canon_sha256":"140b778a599ca0e50c83e5e43475c85490a10a5a818901db2c955f2b9633cbbd"},"schema_version":"1.0","source":{"id":"2408.11061","kind":"arxiv","version":1}},"canonical_sha256":"0c2b907cfd0a5e31ec96b3dcfc51b93e81fbaf3d730d3bcdddd46affd0268332","receipt":{"algorithm":"ed25519","builder_version":"pith-number-builder-2026-05-17-v1","canonical_sha256":"0c2b907cfd0a5e31ec96b3dcfc51b93e81fbaf3d730d3bcdddd46affd0268332","first_computed_at":"2026-07-05T08:57:24.482238Z","key_id":"pith-v1-2026-05","kind":"pith_receipt","last_reissued_at":"2026-07-05T08:57:24.482238Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","receipt_version":"0.3","signature_b64":"miB79lpct4pHmP4PRQQkZZCkt+HA5h/qkFQ0LVPw4xfJMp4pYwEGjRd84LzgtIP65TfvJ3VEIRUNoMy8kIM4Dg==","signature_status":"signed_v1","signed_at":"2026-07-05T08:57:24.482675Z","signed_message":"canonical_sha256_bytes"},"source_id":"2408.11061","source_kind":"arxiv","source_version":1}}},"equivocations":[],"invalid_events":[],"applied_event_ids":["sha256:fb009d2a93a695864033b3943f3fdfaa02763d576fe930e669236ca489f72c4e","sha256:a43a2cbacc3dee92036ab26004007fab97ad85db76463b8f242b1288131187db"],"state_sha256":"50f02352e63d1f326d21c5dc2a393fe2b3f1dc8f3c500608ea21084bdd3b9db2"},"bundle_signature":{"signature_status":"signed_v1","algorithm":"ed25519","key_id":"pith-v1-2026-05","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54","signature_b64":"Pw/cSi3MxocZc5sbE4KUxELArRm4veeZWnisfwFOAHm4JofPOt1Uv2MqRR3zCS+rS+Cutpl0Oyrdv40K/6sCCQ==","signed_message":"bundle_sha256_bytes","signed_at":"2026-08-09T09:46:40.656308Z","bundle_sha256":"f7cffae44a09c5113634e3955a70ab7b94a70266245d02836d3643429ad8c9a5"}}