{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2024:BQVZA7H5BJPDD3EWWPOPYUNZH2","short_pith_number":"pith:BQVZA7H5","schema_version":"1.0","canonical_sha256":"0c2b907cfd0a5e31ec96b3dcfc51b93e81fbaf3d730d3bcdddd46affd0268332","source":{"kind":"arxiv","id":"2408.11061","version":1},"attestation_state":"computed","paper":{"title":"StructuredRAG: JSON Response Formatting with Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Akanksha Sharma, Bob van Luijt, Charles Pierse, Connor Shorten, Erika Cardenas, John Trengrove, Thomas Benjamin Smith","submitted_at":"2024-08-07T19:32:59Z","abstract_excerpt":"The ability of Large Language Models (LLMs) to generate structured outputs, such as JSON, is crucial for their use in Compound AI Systems. However, evaluating and improving this capability remains challenging. In this work, we introduce StructuredRAG, a benchmark of six tasks designed to assess LLMs' proficiency in following response format instructions. We evaluate two state-of-the-art LLMs, Gemini 1.5 Pro and Llama 3 8B-instruct with 4-bit quantization using two distinct prompting strategies. We introduce these prompting strategies as f-String and Follow the Format (FF) prompting. Across 24 "},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2408.11061","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2024-08-07T19:32:59Z","cross_cats_sorted":[],"title_canon_sha256":"140b778a599ca0e50c83e5e43475c85490a10a5a818901db2c955f2b9633cbbd","abstract_canon_sha256":"cd8b242f495f819fab4be5f3b587097eb19771be93b8f8e81e219e85345ef268"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T08:57:24.482675Z","signature_b64":"miB79lpct4pHmP4PRQQkZZCkt+HA5h/qkFQ0LVPw4xfJMp4pYwEGjRd84LzgtIP65TfvJ3VEIRUNoMy8kIM4Dg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0c2b907cfd0a5e31ec96b3dcfc51b93e81fbaf3d730d3bcdddd46affd0268332","last_reissued_at":"2026-07-05T08:57:24.482238Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T08:57:24.482238Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"StructuredRAG: JSON Response Formatting with Large Language Models","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Akanksha Sharma, Bob van Luijt, Charles Pierse, Connor Shorten, Erika Cardenas, John Trengrove, Thomas Benjamin Smith","submitted_at":"2024-08-07T19:32:59Z","abstract_excerpt":"The ability of Large Language Models (LLMs) to generate structured outputs, such as JSON, is crucial for their use in Compound AI Systems. However, evaluating and improving this capability remains challenging. In this work, we introduce StructuredRAG, a benchmark of six tasks designed to assess LLMs' proficiency in following response format instructions. We evaluate two state-of-the-art LLMs, Gemini 1.5 Pro and Llama 3 8B-instruct with 4-bit quantization using two distinct prompting strategies. We introduce these prompting strategies as f-String and Follow the Format (FF) prompting. Across 24 "},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2408.11061","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2408.11061/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2408.11061","created_at":"2026-07-05T08:57:24.482289+00:00"},{"alias_kind":"arxiv_version","alias_value":"2408.11061v1","created_at":"2026-07-05T08:57:24.482289+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2408.11061","created_at":"2026-07-05T08:57:24.482289+00:00"},{"alias_kind":"pith_short_12","alias_value":"BQVZA7H5BJPD","created_at":"2026-07-05T08:57:24.482289+00:00"},{"alias_kind":"pith_short_16","alias_value":"BQVZA7H5BJPDD3EW","created_at":"2026-07-05T08:57:24.482289+00:00"},{"alias_kind":"pith_short_8","alias_value":"BQVZA7H5","created_at":"2026-07-05T08:57:24.482289+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":8,"internal_anchor_count":0,"sample":[{"citing_arxiv_id":"2606.09395","citing_title":"Empirical Study for Structured Output Control in LLMs for Software Engineering","ref_index":28,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14113","citing_title":"ProtoMedAgent: Multimodal Clinical Interpretability via Privacy-Aware Agentic Workflows","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2509.09794","citing_title":"Synthetic Homes: A Multimodal Generative AI Pipeline for Residential Building Data Generation under Data Scarcity","ref_index":40,"is_internal_anchor":false},{"citing_arxiv_id":"2512.09629","citing_title":"End-to-end PDDL Planning with Hardcoded and Dynamic Agents","ref_index":29,"is_internal_anchor":false},{"citing_arxiv_id":"2605.14113","citing_title":"ProtoMedAgent: Multimodal Clinical Interpretability via Privacy-Aware Agentic Workflows","ref_index":39,"is_internal_anchor":false},{"citing_arxiv_id":"2604.14167","citing_title":"Chinese Essay Rhetoric Recognition Using LoRA, In-context Learning and Model Ensemble","ref_index":7,"is_internal_anchor":false},{"citing_arxiv_id":"2604.23059","citing_title":"Implicit Framing in Obstetric Counseling Notes: A Grounded LLM Pipeline on a VBAC-Eligible Cohort","ref_index":50,"is_internal_anchor":false},{"citing_arxiv_id":"2604.20811","citing_title":"Diagnosing CFG Interpretation in LLMs","ref_index":25,"is_internal_anchor":false}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2","json":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2.json","graph_json":"https://pith.science/api/pith-number/BQVZA7H5BJPDD3EWWPOPYUNZH2/graph.json","events_json":"https://pith.science/api/pith-number/BQVZA7H5BJPDD3EWWPOPYUNZH2/events.json","paper":"https://pith.science/paper/BQVZA7H5"},"agent_actions":{"view_html":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2","download_json":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2.json","view_paper":"https://pith.science/paper/BQVZA7H5","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2408.11061&json=true","fetch_graph":"https://pith.science/api/pith-number/BQVZA7H5BJPDD3EWWPOPYUNZH2/graph.json","fetch_events":"https://pith.science/api/pith-number/BQVZA7H5BJPDD3EWWPOPYUNZH2/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/action/timestamp_anchor","attest_storage":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/action/storage_attestation","attest_author":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/action/author_attestation","sign_citation":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/action/citation_signature","submit_replication":"https://pith.science/pith/BQVZA7H5BJPDD3EWWPOPYUNZH2/action/replication_record"}},"created_at":"2026-07-05T08:57:24.482289+00:00","updated_at":"2026-07-05T08:57:24.482289+00:00"}