{"record_type":"pith_number_record","schema_url":"https://pith.science/schemas/pith-number/v1.json","pith_number":"pith:2023:7XGJIMTRRFDVSLSXNXFFTVUAHT","short_pith_number":"pith:7XGJIMTR","schema_version":"1.0","canonical_sha256":"fdcc9432718947592e576dca59d6803cc48b63887b0aa49e482b2949d5a65965","source":{"kind":"arxiv","id":"2309.06009","version":1},"attestation_state":"computed","paper":{"title":"Content Reduction, Surprisal and Information Density Estimation for Long Documents","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Pekka Marttinen, Shaoxiong Ji, Wei Sun","submitted_at":"2023-09-12T07:08:22Z","abstract_excerpt":"Many computational linguistic methods have been proposed to study the information content of languages. We consider two interesting research questions: 1) how is information distributed over long documents, and 2) how does content reduction, such as token selection and text summarization, affect the information density in long documents. We present four criteria for information density estimation for long documents, including surprisal, entropy, uniform information density, and lexical density. Among those criteria, the first three adopt the measures from information theory. We propose an atte"},"verification_status":{"content_addressed":true,"pith_receipt":true,"author_attested":false,"weak_author_claims":0,"strong_author_claims":0,"externally_anchored":false,"storage_verified":false,"citation_signatures":0,"replication_records":0,"graph_snapshot":true,"references_resolved":false,"formal_links_present":false},"canonical_record":{"source":{"id":"2309.06009","kind":"arxiv","version":1},"metadata":{"license":"http://creativecommons.org/licenses/by/4.0/","primary_cat":"cs.CL","submitted_at":"2023-09-12T07:08:22Z","cross_cats_sorted":[],"title_canon_sha256":"4eb05fe88aeb4dc26d42be98373422b3f0599700db7c129cef4caeb6d1eabefd","abstract_canon_sha256":"15b4f55231d062e033d6db5abc6f63bde51974ddaa4fb393865878aa056b9de8"},"schema_version":"1.0"},"receipt":{"kind":"pith_receipt","key_id":"pith-v1-2026-05","algorithm":"ed25519","signed_at":"2026-07-05T06:49:49.948349Z","signature_b64":"25GwxwBzyMzOBh+NPmO1O1PLZuUkZB7XUrRV3RE/r96neQfT7y01X8uZTTdr67fDoib77qDzFT4AAF149L+SCg==","signed_message":"canonical_sha256_bytes","builder_version":"pith-number-builder-2026-05-17-v1","receipt_version":"0.3","canonical_sha256":"fdcc9432718947592e576dca59d6803cc48b63887b0aa49e482b2949d5a65965","last_reissued_at":"2026-07-05T06:49:49.947923Z","signature_status":"signed_v1","first_computed_at":"2026-07-05T06:49:49.947923Z","public_key_fingerprint":"8d4b5ee74e4693bcd1df2446408b0d54"},"graph_snapshot":{"paper":{"title":"Content Reduction, Surprisal and Information Density Estimation for Long Documents","license":"http://creativecommons.org/licenses/by/4.0/","headline":"","cross_cats":[],"primary_cat":"cs.CL","authors_text":"Pekka Marttinen, Shaoxiong Ji, Wei Sun","submitted_at":"2023-09-12T07:08:22Z","abstract_excerpt":"Many computational linguistic methods have been proposed to study the information content of languages. We consider two interesting research questions: 1) how is information distributed over long documents, and 2) how does content reduction, such as token selection and text summarization, affect the information density in long documents. We present four criteria for information density estimation for long documents, including surprisal, entropy, uniform information density, and lexical density. Among those criteria, the first three adopt the measures from information theory. We propose an atte"},"claims":{"count":0,"items":[],"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"source":{"id":"2309.06009","kind":"arxiv","version":1},"verdict":{"id":null,"model_set":{},"created_at":null,"strongest_claim":"","one_line_summary":"","pipeline_version":null,"weakest_assumption":"","pith_extraction_headline":""},"integrity":{"clean":true,"summary":{"advisory":0,"critical":0,"by_detector":{},"informational":0},"endpoint":"/pith/2309.06009/integrity.json","findings":[],"available":true,"detectors_run":[],"snapshot_sha256":"c28c3603d3b5d939e8dc4c7e95fa8dfce3d595e45f758748cecf8e644a296938"},"references":{"count":0,"sample":[],"resolved_work":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57","internal_anchors":0},"formal_canon":{"evidence_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"author_claims":{"count":0,"strong_count":0,"snapshot_sha256":"258153158e38e3291e3d48162225fcdb2d5a3ed65a07baac614ab91432fd4f57"},"builder_version":"pith-number-builder-2026-05-17-v1"},"aliases":[{"alias_kind":"arxiv","alias_value":"2309.06009","created_at":"2026-07-05T06:49:49.947981+00:00"},{"alias_kind":"arxiv_version","alias_value":"2309.06009v1","created_at":"2026-07-05T06:49:49.947981+00:00"},{"alias_kind":"doi","alias_value":"10.48550/arxiv.2309.06009","created_at":"2026-07-05T06:49:49.947981+00:00"},{"alias_kind":"pith_short_12","alias_value":"7XGJIMTRRFDV","created_at":"2026-07-05T06:49:49.947981+00:00"},{"alias_kind":"pith_short_16","alias_value":"7XGJIMTRRFDVSLSX","created_at":"2026-07-05T06:49:49.947981+00:00"},{"alias_kind":"pith_short_8","alias_value":"7XGJIMTR","created_at":"2026-07-05T06:49:49.947981+00:00"}],"events":[],"event_summary":{},"paper_claims":[],"inbound_citations":{"count":1,"internal_anchor_count":1,"sample":[{"citing_arxiv_id":"2412.02149","citing_title":"Leveraging Large Language Models for Comparative Literature Summarization with Reflective Incremental Mechanisms","ref_index":11,"is_internal_anchor":true}]},"formal_canon":{"evidence_count":0,"sample":[],"anchors":[]},"links":{"html":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT","json":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT.json","graph_json":"https://pith.science/api/pith-number/7XGJIMTRRFDVSLSXNXFFTVUAHT/graph.json","events_json":"https://pith.science/api/pith-number/7XGJIMTRRFDVSLSXNXFFTVUAHT/events.json","paper":"https://pith.science/paper/7XGJIMTR"},"agent_actions":{"view_html":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT","download_json":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT.json","view_paper":"https://pith.science/paper/7XGJIMTR","resolve_alias":"https://pith.science/api/pith-number/resolve?arxiv=2309.06009&json=true","fetch_graph":"https://pith.science/api/pith-number/7XGJIMTRRFDVSLSXNXFFTVUAHT/graph.json","fetch_events":"https://pith.science/api/pith-number/7XGJIMTRRFDVSLSXNXFFTVUAHT/events.json","actions":{"anchor_timestamp":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT/action/timestamp_anchor","attest_storage":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT/action/storage_attestation","attest_author":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT/action/author_attestation","sign_citation":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT/action/citation_signature","submit_replication":"https://pith.science/pith/7XGJIMTRRFDVSLSXNXFFTVUAHT/action/replication_record"}},"created_at":"2026-07-05T06:49:49.947981+00:00","updated_at":"2026-07-05T06:49:49.947981+00:00"}