{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2025:CLU7EPMCBJJ647UCBQI7OEMRT5","short_pith_number":"pith:CLU7EPMC","schema_version":"1.0","canonical_sha256":"12e9f23d820a53ee7e820c11f711919f66eb2d7470b313ccda5f5d19abe30841","source":{"kind":"arxiv","id":"2510.10315","version":3},"attestation_state":"computed","paper":{"title":"Is Misinformation More Open? A Study of robots.txt Gatekeeping on the Web","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CY","authors_text":"Devashish Gosain, Ha Dao, Nicolas Steinacker-Olsztyn","submitted_at":"2025-10-11T18:47:14Z","abstract_excerpt":"Large Language Models (LLMs) are increasingly relying on web crawling to stay up to date and accurately answer user queries. These crawlers are expected to honor robots.txt files, which govern automated access. In this study, for the first time, we investigate whether reputable news websites and misinformation sites differ in how they configure these files, particularly in relation to AI crawlers. Analyzing a curated dataset, we find a stark contrast: 60.0% of reputable sites disallow at least one AI crawler, compared to just 9.1% of misinformation sites in their robots.txt files. Reputable si"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2510.10315","kind":"arxiv","version":3},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CY","submitted_at":"2025-10-11T18:47:14Z","cross_cats_sorted":[],"title_canon_sha256":"6e4d904dcb587badfab58b49fdd6decf851377a9cf92345333df617223738660","abstract_canon_sha256":"0708585bcbe0a152d19f25453d7fcebb75dd395958215bd4c5a5af374854daa6"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-29T01:25:29.710241Z","signature_b64":"J7tIEMgzeMR6W4C0yDXibrElHJprK1rPdvLuD7+gZfyPZx5z/Q3qVYRoEVzD9k1vEER7GMgO/Npk/SewVDk7DA==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"12e9f23d820a53ee7e820c11f711919f66eb2d7470b313ccda5f5d19abe30841","last_reissued_at":"2026-07-29T01:25:29.709347Z","signature_status":"signed_v1","first_computed_at":"2026-07-29T01:25:29.709347Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Is Misinformation More Open? A Study of robots.txt Gatekeeping on the Web","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CY","authors_text":"Devashish Gosain, Ha Dao, Nicolas Steinacker-Olsztyn","submitted_at":"2025-10-11T18:47:14Z","abstract_excerpt":"Large Language Models (LLMs) are increasingly relying on web crawling to stay up to date and accurately answer user queries. These crawlers are expected to honor robots.txt files, which govern automated access. In this study, for the first time, we investigate whether reputable news websites and misinformation sites differ in how they configure these files, particularly in relation to AI crawlers. Analyzing a curated dataset, we find a stark contrast: 60.0% of reputable sites disallow at least one AI crawler, compared to just 9.1% of misinformation sites in their robots.txt files. Reputable si"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2510.10315","kind":"arxiv","version":3},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2510.10315/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2510.10315","created_at":"2026-07-29T01:25:29.709764+00:00"},{"alias_kind":"arxiv_version","alias_value":"2510.10315v3","created_at":"2026-07-29T01:25:29.709764+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2510.10315","created_at":"2026-07-29T01:25:29.709764+00:00"},{"alias_kind":"pith_short_12","alias_value":"CLU7EPMCBJJ6","created_at":"2026-07-29T01:25:29.709764+00:00"},{"alias_kind":"pith_short_16","alias_value":"CLU7EPMCBJJ647UC","created_at":"2026-07-29T01:25:29.709764+00:00"},{"alias_kind":"pith_short_8","alias_value":"CLU7EPMC","created_at":"2026-07-29T01:25:29.709764+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2605.23684","citing_title":"Synthetic Sources?: Auditing Generative Search Engine Citations for Evidence of AI-Generated Sources","ref_index":87,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5","json":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5.json","graph_json":"https://pith.science/api/pith-number/CLU7EPMCBJJ647UCBQI7OEMRT5/graph.json","events_json":"https://pith.science/api/pith-number/CLU7EPMCBJJ647UCBQI7OEMRT5/events.json","paper":"https://pith.science/paper/CLU7EPMC"},"agent_actions":{"view_html":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5","download_json":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5.json","view_paper":"https://pith.science/paper/CLU7EPMC","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2510.10315&json=true","fetch_graph":"https://pith.science/api/pith-number/CLU7EPMCBJJ647UCBQI7OEMRT5/graph.json","fetch_events":"https://pith.science/api/pith-number/CLU7EPMCBJJ647UCBQI7OEMRT5/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5/action/timestamp_anchor","attest_storage":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5/action/storage_attestation","attest_author":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5/action/author_attestation","sign_citation":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5/action/citation_signature","submit_replication":"https://pith.science/pith/CLU7EPMCBJJ647UCBQI7OEMRT5/action/replication_record"}},"created_at":"2026-07-29T01:25:29.709764+00:00","updated_at":"2026-07-29T01:25:29.709764+00:00"}