{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:BXDLFMW6Z2IX5JOTDGWQLTWELF","short_pith_number":"pith:BXDLFMW6","schema_version":"1.0","canonical_sha256":"0dc6b2b2dece917ea5d319ad05cec45974351fe418a5199bab485417ccd711d7","source":{"kind":"arxiv","id":"2507.15715","version":2},"attestation_state":"computed","paper":{"title":"From Queries to Criteria: Understanding How Astronomers Evaluate LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["astro-ph.IM"],"primary_cat":"cs.CL","authors_text":"Alina Hyk, Anjalie Field, Ioana Ciuc\\u{a}, J. E. G. Peek, John F Wu, Kartheik G. Iyer, Kiera McCormick, Mian Zhong, Sanjib Sharma, Ziang Xiao","submitted_at":"2025-07-21T15:26:58Z","abstract_excerpt":"There is growing interest in leveraging LLMs to aid in astronomy and other scientific research, but benchmarks for LLM evaluation in general have not kept pace with the increasingly diverse ways that real people evaluate and use these models. In this study, we seek to improve evaluation procedures by building an understanding of how users evaluate LLMs. We focus on a particular use case: an LLM-powered retrieval-augmented generation bot for engaging with astronomical literature, which we deployed via Slack. Our inductive coding of 368 queries to the bot over four weeks and our follow-up interv"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2507.15715","kind":"arxiv","version":2},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2025-07-21T15:26:58Z","cross_cats_sorted":["astro-ph.IM"],"title_canon_sha256":"116fdd374d3ef18c0901d1b7da3fd7131adc2cc3f00e770f963237d004b82c7c","abstract_canon_sha256":"ef31d2aa4609275007247e4aa5a54995c1f991ff30f0b4ccb508f896fb7df409"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T11:49:11.614371Z","signature_b64":"JtWMyGj72S4H4ZAHWflcen85Nl8v0QUudgZtIjpshKl8osSD194S+ah9m/UNHsaaV48IMFn5PVoutDBukuAIAA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"0dc6b2b2dece917ea5d319ad05cec45974351fe418a5199bab485417ccd711d7","last_reissued_at":"2026-07-05T11:49:11.613920Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T11:49:11.613920Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"From Queries to Criteria: Understanding How Astronomers Evaluate LLMs","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":["astro-ph.IM"],"primary_cat":"cs.CL","authors_text":"Alina Hyk, Anjalie Field, Ioana Ciuc\\u{a}, J. E. G. Peek, John F Wu, Kartheik G. Iyer, Kiera McCormick, Mian Zhong, Sanjib Sharma, Ziang Xiao","submitted_at":"2025-07-21T15:26:58Z","abstract_excerpt":"There is growing interest in leveraging LLMs to aid in astronomy and other scientific research, but benchmarks for LLM evaluation in general have not kept pace with the increasingly diverse ways that real people evaluate and use these models. In this study, we seek to improve evaluation procedures by building an understanding of how users evaluate LLMs. We focus on a particular use case: an LLM-powered retrieval-augmented generation bot for engaging with astronomical literature, which we deployed via Slack. Our inductive coding of 368 queries to the bot over four weeks and our follow-up interv"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2507.15715","kind":"arxiv","version":2},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2507.15715/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2507.15715","created_at":"2026-07-05T11:49:11.613976+00:00"},{"alias_kind":"arxiv_version","alias_value":"2507.15715v2","created_at":"2026-07-05T11:49:11.613976+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2507.15715","created_at":"2026-07-05T11:49:11.613976+00:00"},{"alias_kind":"pith_short_12","alias_value":"BXDLFMW6Z2IX","created_at":"2026-07-05T11:49:11.613976+00:00"},{"alias_kind":"pith_short_16","alias_value":"BXDLFMW6Z2IX5JOT","created_at":"2026-07-05T11:49:11.613976+00:00"},{"alias_kind":"pith_short_8","alias_value":"BXDLFMW6","created_at":"2026-07-05T11:49:11.613976+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":0,"internal_anchor_count":0,"sample":[]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF","json":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF.json","graph_json":"https://pith.science/api/pith-number/BXDLFMW6Z2IX5JOTDGWQLTWELF/graph.json","events_json":"https://pith.science/api/pith-number/BXDLFMW6Z2IX5JOTDGWQLTWELF/events.json","paper":"https://pith.science/paper/BXDLFMW6"},"agent_actions":{"view_html":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF","download_json":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF.json","view_paper":"https://pith.science/paper/BXDLFMW6","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2507.15715&json=true","fetch_graph":"https://pith.science/api/pith-number/BXDLFMW6Z2IX5JOTDGWQLTWELF/graph.json","fetch_events":"https://pith.science/api/pith-number/BXDLFMW6Z2IX5JOTDGWQLTWELF/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF/action/timestamp_anchor","attest_storage":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF/action/storage_attestation","attest_author":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF/action/author_attestation","sign_citation":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF/action/citation_signature","submit_replication":"https://pith.science/pith/BXDLFMW6Z2IX5JOTDGWQLTWELF/action/replication_record"}},"created_at":"2026-07-05T11:49:11.613976+00:00","updated_at":"2026-07-05T11:49:11.613976+00:00"}