{"as_of":"2026-08-09T15:38:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:5451b67fa493030779a78b19d62b5f03ea86655cc17ae1d53452f83d37e2ede3","coverage":[{"denominator":46,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":46,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-06T05:48:02.907156Z","state":"measured"},{"denominator":46,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":46,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-09T06:31:02.800959+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2508.01324/citation-record","integrity":"/paper/2508.01324/integrity","json":"/paper/2508.01324/citation-record.json","paper":"/paper/2508.01324"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.693008Z","title":"Sulla determinazione empirica di una legge didistribuzione","venue":null,"work_id":"7d1a6d30-2320-4030-ac17-bd5dcdff7784","year":1933},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.523221Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:63ae869683dae4e8bf2af3114237e898d06205eb65351127d6e03437bd61e341","observation_id":"5db9453e-bb15-42e4-888c-32fcae6c3ed6","resolution":{"observed_at":"2026-08-06T05:48:03.697474Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.529864Z","title":"Machine unlearning","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.529864Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:60b6d7e03a87d0d3790a6bd1210c09f32b285ae2a84f9f138c0f5f1281117ef8","observation_id":"d979d22c-1367-4d60-963a-c3c9a6325924","resolution":{"observed_at":"2026-08-06T05:48:02.529864Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.534875Z","title":"Extracting training data from large language models","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.534875Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:4688af42b0a897f1af75994b7635a63f37cc8deaaace68463b898fe5560b7cdd","observation_id":"b4656c86-009d-4319-af1b-fa78faa77834","resolution":{"observed_at":"2026-08-06T05:48:02.534875Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.20150","last_updated":"2023-10-31T03:35:59Z","snapshot_observed_at":"2026-08-05T03:18:20.348553Z","submitted_at":"2023-10-31T03:35:59Z","title":"Unlearn What You Want to Forget: Efficient Unlearning for LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.20150","snapshot_observed_at":"2026-08-06T05:48:02.539689Z","title":"Unlearn what you want to forget: Efficient unlearning for llms, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.539689Z"},"links":{"cited_paper":"/paper/2310.20150","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:1a7ed87253dd4d4294a74508095dd1853fe02aabe29e775bd64d80fa7f4d0e6a","observation_id":"d8baadf5-d4a0-44f1-8adc-2e4bd587d0f0","resolution":{"observed_at":"2026-08-06T05:48:02.539689Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.544965Z","title":"Simplicity prevails: Rethinking negative preference optimization for llm unlearning, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.544965Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:5e23e0e1526a3072630ab3273ce58f058a432e7c8faed8f77beb5857ba3c0ff2","observation_id":"7e6c7a0d-a9dc-455d-b2b3-ad61c5568607","resolution":{"observed_at":"2026-08-06T05:48:02.544965Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.16289","last_updated":"2023-09-28T09:35:59Z","snapshot_observed_at":"2026-08-07T18:48:25.719201Z","submitted_at":"2023-09-28T09:35:59Z","title":"LawBench: Benchmarking Legal Knowledge of Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.16289","snapshot_observed_at":"2026-08-06T05:48:02.549792Z","title":"Lawbench: Benchmarking legal knowledge of large language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.549792Z"},"links":{"cited_paper":"/paper/2309.16289","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:f8bbb669ac99e69802ce11f2eb65796aee81c204ad432fe24fed38fa5fc623ca","observation_id":"a5bd44a8-2f20-4a47-80a5-fa635c0e9631","resolution":{"observed_at":"2026-08-06T05:48:02.549792Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.658573Z","title":"Ticketed learning–unlearning schemes","venue":null,"work_id":"aeba5c6b-871c-4945-8592-18b43815f4fe","year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.555367Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:b2930c0b5478f1b2863e223d2fa339882a010a04c944a9691410d35a566c1068","observation_id":"414fd268-5811-4931-9a20-c655fd4f282f","resolution":{"observed_at":"2026-08-06T05:48:03.663147Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.559890Z","title":"Making ai forget you: Data deletion in machine learning","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.559890Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:8df2cc0f1c2fa0d847d2b174a3caa5742bead35e6f1665f112978e00d2d25d97","observation_id":"4dace75a-b854-4480-ad13-d80aaf9291e1","resolution":{"observed_at":"2026-08-06T05:48:02.559890Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.632571Z","title":"Adaptive machine unlearning","venue":null,"work_id":"e3a9cc7c-b11c-43c0-924a-598bfaa73faa","year":2021},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.564416Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:b92d18403b2d0a59c9948b53b46212d9754fd6eb1c878ec6f63441126827d6cd","observation_id":"f1ca4b13-82fd-45de-8ff3-f4c5a1143e78","resolution":{"observed_at":"2026-08-06T05:48:03.637383Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.14526","last_updated":"2025-03-26T12:21:42Z","snapshot_observed_at":"2026-08-07T07:47:44.333099Z","submitted_at":"2024-06-20T17:38:16Z","title":"Fantastic Copyrighted Beasts and How (Not) to Generate Them","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.14526","snapshot_observed_at":"2026-08-06T05:48:02.569006Z","title":"Fantastic copyrighted beasts and how (not) to generate them","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.569006Z"},"links":{"cited_paper":"/paper/2406.14526","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:34f5a5c7fd83c3d3d5200dd4e8c2dcb77ba55cb457130849510e8fd8d2d6c439","observation_id":"f5f5ad93-e668-43b0-851b-f4cb62943f08","resolution":{"observed_at":"2026-08-06T05:48:02.569006Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2303.15715","last_updated":"2023-03-28T03:58:40Z","snapshot_observed_at":"2026-07-06T15:08:45.214225Z","submitted_at":"2023-03-28T03:58:40Z","title":"Foundation Models and Fair Use","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.15715","snapshot_observed_at":"2026-08-06T05:48:02.573991Z","title":"Foundation models and fair use","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.573991Z"},"links":{"cited_paper":"/paper/2303.15715","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:9c13f1a814552505d329ba789237b515924f53ec9056501c038d2303b8d8beb9","observation_id":"931bf2cb-d779-4145-b4f9-d0b09616ac81","resolution":{"observed_at":"2026-08-06T05:48:02.573991Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.13356","last_updated":"2025-03-17T07:46:49Z","snapshot_observed_at":"2026-08-01T17:13:21.519528Z","submitted_at":"2024-06-19T09:03:21Z","title":"Unlearning or Obfuscating? Jogging the Memory of Unlearned LLMs via Benign Relearning","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.13356","snapshot_observed_at":"2026-08-06T05:48:02.579035Z","title":"Jogging the memory of unlearned llms through targeted relearning attacks, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.579035Z"},"links":{"cited_paper":"/paper/2406.13356","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:5db6bad32b67d8fe0f8f08d150ce1ec06d274310e07ed15897f72ad2a36c0b25","observation_id":"953de8b7-c633-420b-adbb-3e320365b80a","resolution":{"observed_at":"2026-08-06T05:48:02.579035Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.583899Z","title":"Eraser: Machine unlearning in mlaas via an inference serving-aware approach","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.583899Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:d70466eea75892e15ae14578e913a7408258c8c77a485d61fd23ecf1a9926350","observation_id":"c7211c40-63de-445f-997a-d3c405bfb183","resolution":{"observed_at":"2026-08-06T05:48:02.583899Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.588391Z","title":"Editing models with task arithmetic, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.588391Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:8dfd386750f35184161822c06be90c35cafcb3238644cd1b0e8d3a8bfa6031c8","observation_id":"7c87c9ab-950b-4559-87fd-970441babda6","resolution":{"observed_at":"2026-08-06T05:48:02.588391Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.598200Z","title":"Knowledge sanitization of large language models,","venue":null,"work_id":"9030ef69-fad6-412c-a4cb-044ba44857cf","year":null},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.592743Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:7a2079fb9a3c2ea0ed5cd7f204b8287e990d35325bf54570daa962beb1587bd6","observation_id":"adfa6778-831f-46e8-9ac3-cd8d7a9abc48","resolution":{"observed_at":"2026-08-06T05:48:03.602515Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.602641Z","title":"Approximate data deletion from machine learning models","venue":null,"work_id":null,"year":2008},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.602641Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:9949afb39178ae2b3b3e34b05bac1199a006ef07743cfe03816dfb973ceb3a3f","observation_id":"e7518439-8fb7-4301-bc7a-1f1b8a9c1394","resolution":{"observed_at":"2026-08-06T05:48:02.602641Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2210.01504","last_updated":"2022-12-19T11:52:40Z","snapshot_observed_at":"2026-08-09T15:28:56.392520Z","submitted_at":"2022-10-04T10:18:11Z","title":"Knowledge Unlearning for Mitigating Privacy Risks in Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2210.01504","snapshot_observed_at":"2026-08-06T05:48:02.607226Z","title":"Knowledge unlearning for mitigating privacy risks in language models","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.607226Z"},"links":{"cited_paper":"/paper/2210.01504","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:3c274c3885ec200323546b1dafe8e88c5d037b79dd38ab54181528ef47ee0610","observation_id":"516c5bd1-6858-49d0-82e0-5c232f8ae232","resolution":{"observed_at":"2026-08-06T05:48:02.607226Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.10890","last_updated":"2024-06-16T10:47:21Z","snapshot_observed_at":"2026-08-03T22:46:04.459058Z","submitted_at":"2024-06-16T10:47:21Z","title":"RWKU: Benchmarking Real-World Knowledge Unlearning for Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.10890","snapshot_observed_at":"2026-08-06T05:48:02.612369Z","title":"Rwku: Benchmarking real-world knowledge unlearning for large language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.612369Z"},"links":{"cited_paper":"/paper/2406.10890","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:f9b0a589efb540e2967e83ab071f6b3a28ac2d59df1b4cca1568f69409ee6bd6","observation_id":"f5c278cd-5460-4480-a3e7-98ac5ede2096","resolution":{"observed_at":"2026-08-06T05:48:02.612369Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2212.09573","last_updated":"2022-12-19T16:06:45Z","snapshot_observed_at":"2026-08-05T11:45:14.076085Z","submitted_at":"2022-12-19T16:06:45Z","title":"Privacy Adhering Machine Un-learning in NLP","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.09573","snapshot_observed_at":"2026-08-06T05:48:02.616811Z","title":"Privacy adhering machine un-learning in nlp, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.616811Z"},"links":{"cited_paper":"/paper/2212.09573","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:fcfc791758308b1ce6b4cebc7b9dabb7d66d5b62bfe37b343326767929d8b052","observation_id":"b011107c-7be2-4410-832f-4bba8d4975f7","resolution":{"observed_at":"2026-08-06T05:48:02.616811Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.03218","last_updated":"2024-05-15T19:16:09Z","snapshot_observed_at":"2026-08-09T09:20:14.433688Z","submitted_at":"2024-03-05T18:59:35Z","title":"The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.03218","snapshot_observed_at":"2026-08-06T05:48:02.621729Z","title":"Li, Ann-Kathrin Dombrowski, Shashwat Goel, Long Phan, Gabriel Mukobi, Nathan Helm- Burger, Rassin Lababidi, Lennart Justen, Andrew B","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.621729Z"},"links":{"cited_paper":"/paper/2403.03218","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:f303104adefc4c904f79a6912a892224b340535bc8ebbdd4e0753727a65ff043","observation_id":"154f0426-2040-48d8-817e-9ec239c4edce","resolution":{"observed_at":"2026-08-06T05:48:02.621729Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.573763Z","title":"Large language models in finance: A survey","venue":null,"work_id":"3a8ce9e5-e469-4cdd-aaa4-1e0d1c8bdc8c","year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.626971Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:d57134724bf29d918ce1cf3b9a6539da1dfc467648756a2e0a2cece78f808bb7","observation_id":"334db7be-be5e-4d91-b5ab-aeb8fd449c57","resolution":{"observed_at":"2026-08-06T05:48:03.578378Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.559304Z","title":"Continual learning and private unlearning, 2022","venue":null,"work_id":"5367f652-e827-46b1-ab68-d66b2f54ed91","year":2022},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.631515Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:0e883c5355a5e1875c8d3fe8a8f213912ede168ba44d9f7aac78b82fc3abba8e","observation_id":"db6db72e-136e-461e-aabd-d023f4bef501","resolution":{"observed_at":"2026-08-06T05:48:03.563558Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2012.13891","last_updated":"2021-05-06T04:50:42Z","snapshot_observed_at":"2026-07-06T10:28:00.175226Z","submitted_at":"2020-12-27T08:54:37Z","title":"Federated Unlearning","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2012.13891","snapshot_observed_at":"2026-08-06T05:48:02.639487Z","title":"Federated unlearning","venue":null,"work_id":null,"year":2012},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.639487Z"},"links":{"cited_paper":"/paper/2012.13891","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:97b5cd140458c69d4057ec8b61af7bf5b9a2572e0622edb241012d13605989ae","observation_id":"78c20564-86f0-4dc2-97aa-71b59ff41221","resolution":{"observed_at":"2026-08-06T05:48:02.639487Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.661057Z","title":"Quark: Controllable text generation with reinforced unlearning","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.661057Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:b5f57827e3c586e25703a7d2ff6122ac0287023238d10d06f5184e81deec9a8b","observation_id":"64b8329b-87ff-41b4-8308-57b5c10de8a1","resolution":{"observed_at":"2026-08-06T05:48:02.661057Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.534807Z","title":"Eight methods to evaluate robust unlearning in llms, 2024","venue":null,"work_id":"af7cac49-719e-4ec6-9ab6-c7005a1ba096","year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.685628Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:86cda52a1e5fda5816761b402042fd6e376ef294fbc0cceb2b22c3acdc3479e8","observation_id":"20d47fb7-7c61-430c-9989-063b3b707d9c","resolution":{"observed_at":"2026-08-06T05:48:03.539146Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2401.06121","last_updated":"2024-01-11T18:57:12Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-01-11T18:57:12Z","title":"TOFU: A Task of Fictitious Unlearning for LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.06121","snapshot_observed_at":"2026-08-06T05:48:02.713221Z","title":"Zico Kolter","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.713221Z"},"links":{"cited_paper":"/paper/2401.06121","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:d1269eb09c214c01897a5b22e8a81d75ab0db05c97ae26f51b6284ae24a2c279","observation_id":"a9ac0ed7-b71a-4457-a1d9-95ff66c7ab1d","resolution":{"observed_at":"2026-08-06T05:48:02.713221Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.04430","last_updated":"2024-07-31T02:15:31Z","snapshot_observed_at":"2026-07-06T16:04:02.943124Z","submitted_at":"2023-08-08T17:58:15Z","title":"SILO Language Models: Isolating Legal Risk In a Nonparametric Datastore","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.04430","snapshot_observed_at":"2026-08-06T05:48:02.729818Z","title":"Silo language models: Isolating legal risk in a nonparametric datastore","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.729818Z"},"links":{"cited_paper":"/paper/2308.04430","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:3951ad902506127df71e63d1313f32f82c97ce2e204c0a503337ee11574af12e","observation_id":"12359976-7c7d-4582-8e59-a0f3f0a85231","resolution":{"observed_at":"2026-08-06T05:48:02.729818Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.17410","last_updated":"2023-09-29T17:12:43Z","snapshot_observed_at":"2026-07-06T16:25:38.571377Z","submitted_at":"2023-09-29T17:12:43Z","title":"Can Sensitive Information Be Deleted From LLMs? Objectives for Defending Against Extraction Attacks","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.17410","snapshot_observed_at":"2026-08-06T05:48:02.749611Z","title":"Can sensitive information be deleted from llms? objectives for defending against extraction attacks, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.749611Z"},"links":{"cited_paper":"/paper/2309.17410","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:ad30f89eb50984549e552a6c609d2b3d2a839da4937f3b98c20882811eab0156","observation_id":"683d92f7-89a7-46e7-9dc5-df3e00a8b553","resolution":{"observed_at":"2026-08-06T05:48:02.749611Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.07579","last_updated":"2024-06-06T06:31:08Z","snapshot_observed_at":"2026-07-06T16:31:16.784700Z","submitted_at":"2023-10-11T15:19:31Z","title":"In-Context Unlearning: Language Models as Few Shot Unlearners","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.07579","snapshot_observed_at":"2026-08-06T05:48:02.768760Z","title":"In-context unlearning: Language models as few shot unlearners","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.768760Z"},"links":{"cited_paper":"/paper/2310.07579","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:385b99afc6468f114a8776fc7ec1e18af70d8d906dc686588ac0fd6b36fc38d4","observation_id":"1d9e1ad5-fc32-4ee7-a184-567033d4fd30","resolution":{"observed_at":"2026-08-06T05:48:02.768760Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.786038Z","title":"Manning, and Chelsea Finn","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.786038Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:994b48214cc532c26b5a2015248fd1cdb0d47249285449d4d10915b530c1097b","observation_id":"7ea821e2-976d-4202-903b-75b98fd3a9f6","resolution":{"observed_at":"2026-08-06T05:48:02.786038Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.510947Z","title":"Remember what you want to forget: Algorithms for machine unlearning","venue":null,"work_id":"59d441e3-d20d-4a5a-b389-5e3c4c154f74","year":2021},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.802915Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:b85570c510670d429f2770178be28be230042a308831062da2390b75283137d7","observation_id":"e5aa771c-3f4d-44ce-95b9-c360057529ea","resolution":{"observed_at":"2026-08-06T05:48:03.515596Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.06460","last_updated":"2024-07-14T20:14:02Z","snapshot_observed_at":"2026-07-06T18:43:22.614883Z","submitted_at":"2024-07-08T23:47:29Z","title":"MUSE: Machine Unlearning Six-Way Evaluation for Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.06460","snapshot_observed_at":"2026-08-06T05:48:02.822371Z","title":"Smith, and Chiyuan Zhang","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.822371Z"},"links":{"cited_paper":"/paper/2407.06460","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:2feda1315293a6ec87d3a4b87810c7d78fa70f138372e77cc6df31372c7ffac4","observation_id":"5eb0a70d-e603-4544-9080-e74b651b3c44","resolution":{"observed_at":"2026-08-06T05:48:02.822371Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.496489Z","title":null,"venue":null,"work_id":"20fb3227-5835-4af1-bd2a-e08c8d3cfd5b","year":1939},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.835938Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:be6e2200aa01ec96f3bb0546a7f4dabf5c4099ecf5586cf322d061eab4133db1","observation_id":"b438490b-c157-4b40-9d56-be645ccbc383","resolution":{"observed_at":"2026-08-06T05:48:03.500982Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.482752Z","title":"Chatgpt: Optimizing language models for dialogue","venue":null,"work_id":"1a6e3b38-1696-49dd-9dc7-7afd1fa8fc71","year":2022},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.855006Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:afc94f4378e043aafe8bae2d403c05ec01913194a20ff06de79972bc642da5cb","observation_id":"f8484393-34d5-4b2a-8ea5-aac4c3780606","resolution":{"observed_at":"2026-08-06T05:48:03.486923Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.468385Z","title":"Guardrail baselines for unlearning in llms, 2024","venue":null,"work_id":"f86e6a29-7544-4f42-ac11-697c80d184c8","year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.866575Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:7934d7c1a0116cb83fedce3bf1b46772041d8a771a45352b214cb09554c97894","observation_id":"1d2fe6b2-be13-4a70-9a38-7217bb885fbe","resolution":{"observed_at":"2026-08-06T05:48:03.472838Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.870689Z","title":"Large language models in medicine.Nature medicine, 29(8):1930–1940, 2023","venue":null,"work_id":null,"year":1930},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.870689Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:b3e692a6e4919afd6a1ef4ef66eb165df24631af41e84f214ef78aae7c85021a","observation_id":"7eb13d51-1919-46a2-8ba7-bc4a219f6252","resolution":{"observed_at":"2026-08-06T05:48:02.870689Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.444556Z","title":"Towards efficient and effective unlearning of large language models for recommendation,","venue":null,"work_id":"fe06c975-69f8-4b45-a343-e8da66d8cfa6","year":null},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.874625Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:422053c9ded50d24e5b9f8e181ddf3a23d85c1c498f8d5faabf7603210312e2c","observation_id":"832cc68c-96d2-4ca5-ae96-c5efd9f0ac13","resolution":{"observed_at":"2026-08-06T05:48:03.449164Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.20138","last_updated":"2023-12-05T16:14:24Z","snapshot_observed_at":"2026-07-06T16:40:55.334347Z","submitted_at":"2023-10-31T03:09:36Z","title":"DEPN: Detecting and Editing Privacy Neurons in Pretrained Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.20138","snapshot_observed_at":"2026-08-06T05:48:02.882817Z","title":"Depn: Detecting and editing privacy neurons in pretrained language models","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.882817Z"},"links":{"cited_paper":"/paper/2310.20138","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:bf7ef2f1e91e7993027283e6b5c706fc3a8bd65c2d9029497570e0c2e5119ad9","observation_id":"bc137143-2c23-456c-9ae8-5d7827c67742","resolution":{"observed_at":"2026-08-06T05:48:02.882817Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.03536","last_updated":"2024-06-30T04:00:06Z","snapshot_observed_at":"2026-08-06T13:29:53.985027Z","submitted_at":"2024-03-06T08:31:35Z","title":"Towards Efficient and Effective Unlearning of Large Language Models for Recommendation","version":2},"cited_work":{"arxiv_id":"2403.03536","doi":null,"metadata_source":"pith","pith_arxiv_id":"2403.03536","snapshot_observed_at":"2026-08-06T05:48:02.991121Z","title":"Towards Efficient and Effective Unlearning of Large Language Models for Recommendation","venue":"cs.IR","work_id":"fa3c0a0c-0911-451c-b570-d0eba1836496","year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.878636Z"},"links":{"cited_paper":"/paper/2403.03536","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:8d50e6b79ec0caeca46d20ccc1c25344449950ea27f2eaf67c88ecd3f3d8da18","observation_id":"14fcf720-556e-41d5-9827-a5a6e793be66","resolution":{"observed_at":"2026-08-06T05:48:02.997663Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.416594Z","title":"Evaluating reading comprehension exercises generated by llms: A showcase of chatgpt in education applications","venue":null,"work_id":"bc1769a8-1c22-4b2d-8a04-fdc84a4c3bc1","year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.890665Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:393640a819f0cd70eb6ec3918076352f524501f3f0ca20b5816565e1bbc43f61","observation_id":"5d398bde-1759-462c-b430-fde0b50e9f83","resolution":{"observed_at":"2026-08-06T05:48:03.420844Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.430327Z","title":"Deltagrad: Rapid retraining of machine learning models","venue":null,"work_id":"a75d5005-04b5-49e9-a449-e483191e2c63","year":2020},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.886771Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:70b49d07ed8a817f17ed14ba370bfa2c9bf770d6367672b4a10b20f525e846eb","observation_id":"7ec5b1e5-a801-4576-952a-c1f8445102b6","resolution":{"observed_at":"2026-08-06T05:48:03.434863Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2306.14870","last_updated":"2023-12-09T02:46:08Z","snapshot_observed_at":"2026-08-03T10:10:49.848317Z","submitted_at":"2023-06-26T17:33:21Z","title":"Composing Parameter-Efficient Modules with Arithmetic Operations","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.14870","snapshot_observed_at":"2026-08-06T05:48:02.898919Z","title":"Composing parameter-efficient modules with arithmetic operations, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.898919Z"},"links":{"cited_paper":"/paper/2306.14870","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:3edc52170380562415af28a436f4267b9366a7c79b290e737979455c2c9b8c31","observation_id":"eeb4f3fd-fc45-4d08-be27-f8662f67065e","resolution":{"observed_at":"2026-08-06T05:48:02.898919Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:03.402063Z","title":"Unlearning bias in language models by partitioning gradients","venue":null,"work_id":"a3183a7c-529e-49d6-9f17-06be33dc2790","year":2023},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.894930Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:c3e99e6e689a81aa9709b050061422b3f60c4a3369a11cc98679fb7a0afc686f","observation_id":"84f1df36-2a81-4c12-be8b-b0d979384cdc","resolution":{"observed_at":"2026-08-06T05:48:03.406821Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-09T06:31:02.800959+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.16454","last_updated":"2025-03-21T06:37:37Z","snapshot_observed_at":"2026-07-06T19:37:24.895623Z","submitted_at":"2024-10-21T19:28:37Z","title":"Catastrophic Failure of LLM Unlearning via Quantization","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.16454","snapshot_observed_at":"2026-08-06T05:48:02.907156Z","title":"For the given blank filling question ‘{blankfilling_question}’, the reference material is ‘{answer}’. Your response should only include answers separated by spaces:","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.907156Z"},"links":{"cited_paper":"/paper/2410.16454","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:781c2206660c2c4500951ace36848e1cd578fa544fd695d75e1ae2d35adb0aee","observation_id":"868fca0a-d5a2-4ba7-bdf7-c78b80355a92","resolution":{"observed_at":"2026-08-06T05:48:02.907156Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-06T05:48:02.903079Z","title":"Negative preference optimization: From catastrophic collapse to effective unlearning, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.903079Z"},"links":{"citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:813ee9a13bec833adfdcb02747582168075b2d329d648f1d097a61d48527ec67","observation_id":"96d987cb-32f4-4906-8aa4-dcc6353cb777","resolution":{"observed_at":"2026-08-06T05:48:02.903079Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.11852","last_updated":"2024-03-02T08:50:40Z","snapshot_observed_at":"2026-07-06T16:21:37.457918Z","submitted_at":"2023-09-21T07:49:55Z","title":"Knowledge Sanitization of Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.11852","snapshot_observed_at":"2026-08-06T05:48:02.597724Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-06T05:48:02.597724Z"},"links":{"cited_paper":"/paper/2309.11852","citing_paper":"/paper/2508.01324"},"observation_digest":"sha256:41364c94efb1b8fbd19f9d5e076872cc2a0add67db9e0e11742dcc6ffa4b2969","observation_id":"e9fef472-d2b6-4f4c-b95b-8b5ce9ec8d45","resolution":{"observed_at":"2026-08-06T05:48:02.597724Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2508.01324","last_updated":"2025-08-02T11:32:41Z","latest_version":1,"primary_category":"cs.AI","snapshot_observed_at":"2026-08-08T23:44:32.432109Z","submitted_at":"2025-08-02T11:32:41Z","title":"Towards Evaluation for Real-World LLM Unlearning"},"reference_resolution":{"displayed":46,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":31,"verified_exact":1,"verified_fuzzy":14},"total_outbound_references":46},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-09T06:31:02.800959+00:00","source":"crossref"},{"observed_at":"2026-08-09T06:30:57.326959+00:00","source":"retraction_watch"}],"thesis":"As of 9 August 2026, this Paper Citation Record lists 46 of 46 outbound references and 0 inbound Pith citation observations for arXiv:2508.01324."}