{"as_of":"2026-08-12T18:45:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:b0b05a622dd4602ec5db830966647c725467f68edcbec5427ad384cae47a1238","coverage":[{"denominator":26,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":26,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-08T16:55:24.036678Z","state":"measured"},{"denominator":26,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":26,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-12T06:34:41.77262+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2503.04756/citation-record","integrity":"/paper/2503.04756/integrity","json":"/paper/2503.04756/citation-record.json","paper":"/paper/2503.04756"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.309807Z","title":"45 real-world llm applications and use cases from top companies","venue":null,"work_id":"a6d1278d-4301-4445-8123-60c3ce29bde1","year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.940311Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:5914c8fac1298fcbcedf05973eafa136e400b20b8a54b730e18e5fd626db1bb4","observation_id":"239bd8ca-8c33-418b-a5ee-433ddf6e1ccb","resolution":{"observed_at":"2026-08-08T16:55:24.313635Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-08T16:55:23.945043Z","title":"Training verifiers to solve math word problems","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.945043Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:64e7d1aebc3d315f73e7c8e445b594b1eca88a7cf1124a388bb152b4342ba873","observation_id":"c9de2a6d-db49-47fb-aa19-cca5587c3257","resolution":{"observed_at":"2026-08-08T16:55:23.945043Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.15938","last_updated":"2024-05-31T17:49:03Z","snapshot_observed_at":"2026-08-10T09:23:20.862829Z","submitted_at":"2024-02-24T23:54:41Z","title":"Generalization or Memorization: Data Contamination and Trustworthy Evaluation for Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.15938","snapshot_observed_at":"2026-08-08T16:55:23.949695Z","title":"Generalization or memorization: Data contamination and trustworthy evaluation for large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.949695Z"},"links":{"cited_paper":"/paper/2402.15938","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:5ab23b4d659eafb0d2e0d90d1294fbce7202420fc5de84bbe0ea6f90320d5617","observation_id":"57a481af-8e65-45b4-9092-787870eb4d1e","resolution":{"observed_at":"2026-08-08T16:55:23.949695Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-10T12:35:09.020030Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-08T16:55:23.953730Z","title":"Measuring massive multitask language understanding","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.953730Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:6bb41e8fe4104e85a88cea34c0cea58296f1c399c2b1aeead68ff41e30a3123e","observation_id":"d1496702-94f2-4232-84be-347638243bb9","resolution":{"observed_at":"2026-08-08T16:55:23.953730Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.01748","last_updated":"2024-10-02T17:01:10Z","snapshot_observed_at":"2026-07-06T19:26:22.646942Z","submitted_at":"2024-10-02T17:01:10Z","title":"Not All LLM Reasoners Are Created Equal","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.01748","snapshot_observed_at":"2026-08-08T16:55:23.957746Z","title":"Not all llm reasoners are created equal","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.957746Z"},"links":{"cited_paper":"/paper/2410.01748","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:bae4ec796cbd9e378d04d43572593496bf643bd2893435dce93ec75c340b17c2","observation_id":"5207ff91-b74b-4dba-ac79-bc998566bbe3","resolution":{"observed_at":"2026-08-08T16:55:23.957746Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.299700Z","title":"Hashimoto","venue":null,"work_id":"c29287ae-0225-4e26-9d9e-0c28baa448c6","year":2023},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.961615Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:c319ddc9ae0174c15e0b5118de1586d9eb62823524e85b6c163b5f551619e9b8","observation_id":"1207aca8-1f3f-409d-8ab5-ab4c4d6f0957","resolution":{"observed_at":"2026-08-08T16:55:24.303211Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2408.00118","last_updated":"2024-10-02T15:22:49Z","snapshot_observed_at":"2026-08-02T16:20:09.773989Z","submitted_at":"2024-07-31T19:13:07Z","title":"Gemma 2: Improving Open Language Models at a Practical Size","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.00118","snapshot_observed_at":"2026-08-08T16:55:23.965170Z","title":"Alpacafarm: A simulation framework for methods that learn from human feedback","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.965170Z"},"links":{"cited_paper":"/paper/2408.00118","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:d54f840f9a69aa036233ccf68a4570e18ef2fb0646f8c91a50a47bd745011ace","observation_id":"06105e76-8e3d-4151-8639-f7341c5102ca","resolution":{"observed_at":"2026-08-08T16:55:23.965170Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.289642Z","title":"Transition from online elo rating system to bradley-terry model","venue":null,"work_id":"c642f9f0-b2f9-4c59-90a8-82efeb92b170","year":2023},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.969508Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:52da671c1f0b7969a4bae4805fe3ed370388e7571052cbe2664053a454d28a48","observation_id":"a589bcae-cf43-4269-8d50-cbb55cf2e526","resolution":{"observed_at":"2026-08-08T16:55:24.293300Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.279125Z","title":"Lmsys org","venue":null,"work_id":"0e4a1605-05a3-4cd0-bd89-671f27316085","year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.973267Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:c1762f094f7ce487edbcc68bf51887492840c19bd1d46786866636a2a60e203a","observation_id":"4f510c19-0d8c-465c-85ba-c194dde6fe7a","resolution":{"observed_at":"2026-08-08T16:55:24.282321Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.05229","last_updated":"2025-08-27T16:24:39Z","snapshot_observed_at":"2026-07-06T19:29:09.714725Z","submitted_at":"2024-10-07T17:36:37Z","title":"GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.05229","snapshot_observed_at":"2026-08-08T16:55:23.976970Z","title":"Gsm-symbolic: Understanding the limitations of mathematical reasoning in large language models","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.976970Z"},"links":{"cited_paper":"/paper/2410.05229","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:a23a598000432abdfa35af48817073de854e51527bf90f3280c0d6ec287a2f07","observation_id":"d5614dd6-14a0-4420-b8ea-a9a2ce931695","resolution":{"observed_at":"2026-08-08T16:55:23.976970Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.269195Z","title":"Mistral-7b-v0.3, 2024","venue":null,"work_id":"789ebf5e-3f61-452d-87dc-7a62fd25cc14","year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.981238Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:65d0ab96dc8a02ccaf360c8e1d2540a05aeafd729b1f7576068c7215efdb1c11","observation_id":"ab331eaf-baa7-4892-999e-b503bddd46e3","resolution":{"observed_at":"2026-08-08T16:55:24.272663Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.258884Z","title":"Usa contests","venue":null,"work_id":"ae47dd6f-ecd7-4381-9f5a-d173d25554c5","year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.986768Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:1dfe5852e46d34c712c9260d1b8090f1682ee6a60f741b8ad16f37e7268bbeca","observation_id":"5ffcfe89-3e84-4f25-8e12-14d71f9ffb25","resolution":{"observed_at":"2026-08-08T16:55:24.262528Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.13076","last_updated":"2024-04-15T16:49:59Z","snapshot_observed_at":"2026-08-12T17:44:10.538968Z","submitted_at":"2024-04-15T16:49:59Z","title":"LLM Evaluators Recognize and Favor Their Own Generations","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.13076","snapshot_observed_at":"2026-08-08T16:55:23.990465Z","title":"Llm evaluators recognize and favor their own generations","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.990465Z"},"links":{"cited_paper":"/paper/2404.13076","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:c466cf5e84b0e6fb14c4306d391b2ab4825515ccfdb92e685169ea6558313f98","observation_id":"4270e169-fa7e-4639-aa44-edc42768674f","resolution":{"observed_at":"2026-08-08T16:55:23.990465Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.247462Z","title":"Openai valued at \\ 157 billion after closing \\ 6.6 billion funding round, 10 2024","venue":null,"work_id":"1a977238-327e-4568-b076-df2844028a7a","year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.994127Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:0e691badf5d5229db1d87feb037fdc3606d3cf80c01b24b759d00926e4c30963","observation_id":"f0ea56f9-05b1-47b3-a4e7-9a5932d73f89","resolution":{"observed_at":"2026-08-08T16:55:24.251608Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.237075Z","title":"Top 10 real-life applications of large language models","venue":null,"work_id":"32719aed-4277-4e1f-bb61-30d7c84febcf","year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:23.997551Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:b0d8fe975b6758972c8d770ebbcf7a3d6ad729e3b39fae52ae1ec3a53710ad14","observation_id":"1cd6f61b-e2d0-4f95-8fd7-bee9d3006947","resolution":{"observed_at":"2026-08-08T16:55:24.240177Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1902.10811","last_updated":"2019-06-12T17:42:33Z","snapshot_observed_at":"2026-08-07T19:33:21.437256Z","submitted_at":"2019-02-13T20:35:44Z","title":"Do ImageNet Classifiers Generalize to ImageNet?","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1902.10811","snapshot_observed_at":"2026-08-08T16:55:24.000946Z","title":"Do imagenet classifiers generalize to imagenet? arXiv preprint arXiv:1902.10811, 2019","venue":null,"work_id":null,"year":1902},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.000946Z"},"links":{"cited_paper":"/paper/1902.10811","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:506a1e0f19c53888163e142bf28a6a6280d88d8c2e07a8604f7fbfdfdac24413","observation_id":"182bdf26-23b5-4c64-abe4-724a11f70c6d","resolution":{"observed_at":"2026-08-08T16:55:24.000946Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.226321Z","title":"Nlp evaluation in trouble: On the need to measure llm data contamination for each benchmark","venue":null,"work_id":"da202b35-cbe1-42a0-8e8e-5f3cec85831d","year":2023},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.004742Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:466cfde75b28794ee34758d2ac3e632d19d69bf08caf801ae121c36a7970b736","observation_id":"1bf4bcb4-c507-4bb6-8ebe-7edfac79ba54","resolution":{"observed_at":"2026-08-08T16:55:24.230673Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.216797Z","title":"Language model leaderboard","venue":null,"work_id":"fc0a1bf2-84d8-432c-875b-29fd85e1bcea","year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.008186Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:21cff7c3d7ee116e61d9e00b600366941595792747b38207b24b2a03119efdcb","observation_id":"548c8795-2ba0-4de5-b0d2-e8c9f6077151","resolution":{"observed_at":"2026-08-08T16:55:24.219869Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.207157Z","title":"Taori et al","venue":null,"work_id":"6659da9e-aa14-4b2f-ab37-6580219de1ad","year":2023},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.011598Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:af755e7227dda88c8e2e41352f9f42169f868cac604893380d83c9fd78dcebba","observation_id":"bb8dfb1a-fbc5-4478-b537-97008cb3fe87","resolution":{"observed_at":"2026-08-08T16:55:24.210189Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.01574","last_updated":"2024-11-06T02:54:00Z","snapshot_observed_at":"2026-08-06T00:29:17.674418Z","submitted_at":"2024-06-03T17:53:00Z","title":"MMLU-Pro: A More Robust and Challenging Multi-Task Language Understanding Benchmark","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.01574","snapshot_observed_at":"2026-08-08T16:55:24.015104Z","title":"Mmlu-pro: A more robust and challenging multi-task language understanding benchmark","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.015104Z"},"links":{"cited_paper":"/paper/2406.01574","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:53ce359ceaf7946ccdfe850b0f6f73b3ed30b3f33add91be613989904d562254","observation_id":"1a902257-31c0-44aa-a9fb-0da23d5aba01","resolution":{"observed_at":"2026-08-08T16:55:24.015104Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.193677Z","title":"Chinese wall, wikipedia","venue":null,"work_id":"e4389d1b-1871-4817-ba61-6280da464c2e","year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.018658Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:bfd081472597fced5512cc5a39ae76b9414b70e5926ebe9e39ddd5e271623761","observation_id":"9e3ea815-9f67-4265-a480-8e1fb1ce61a1","resolution":{"observed_at":"2026-08-08T16:55:24.198672Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.00332","last_updated":"2024-11-22T22:27:49Z","snapshot_observed_at":"2026-08-09T21:05:48.710894Z","submitted_at":"2024-05-01T05:52:05Z","title":"A Careful Examination of Large Language Model Performance on Grade School Arithmetic","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.00332","snapshot_observed_at":"2026-08-08T16:55:24.021886Z","title":"A careful examination of large language model performance on grade school arithmetic","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.021886Z"},"links":{"cited_paper":"/paper/2405.00332","citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:3ead34f1314ee17cdce7c7c11c666739da4dc220b62800127966c377eb45d073","observation_id":"2dda6688-f56f-4220-a0f9-58efe9f38463","resolution":{"observed_at":"2026-08-08T16:55:24.021886Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.025418Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.025418Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:5834454ba1df69f68aad24377d3ae46a67fb1d9c9223fa6a7671370cb6112964","observation_id":"b988b621-67d8-4239-beff-005006674baf","resolution":{"observed_at":"2026-08-08T16:55:24.025418Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.029562Z","title":"@esa (Ref","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.029562Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:42610f47ad5f97bc97e0a60afbf99fe7ddf01a9e69d4699f90ffc3c9f375e58a","observation_id":"89b84a08-f80e-4956-ada4-a150c8dd851b","resolution":{"observed_at":"2026-08-08T16:55:24.029562Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.033192Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.033192Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:2721af663488357e14a74a758b7f646afe9ff906ea527a3181ddc72d9bf629e5","observation_id":"915d41d4-33a8-4c29-a72c-8e2329fcec63","resolution":{"observed_at":"2026-08-08T16:55:24.033192Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-08T16:55:24.036678Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-08T16:55:24.036678Z"},"links":{"citing_paper":"/paper/2503.04756"},"observation_digest":"sha256:2d760677701a518194469bc00a308361f7bbdf5a9410b03f312c0278c562fba6","observation_id":"0dbf376f-59eb-4fbc-8e3d-020ce505e295","resolution":{"observed_at":"2026-08-08T16:55:24.036678Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2503.04756","last_updated":"2025-02-09T23:57:33Z","latest_version":1,"primary_category":"cs.CY","snapshot_observed_at":"2026-08-10T03:58:49.986474Z","submitted_at":"2025-02-09T23:57:33Z","title":"Peeking Behind Closed Doors: Risks of LLM Evaluation by Private Data Curators"},"reference_resolution":{"displayed":26,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":14,"verified_exact":0,"verified_fuzzy":12},"total_outbound_references":26},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"thesis":"As of 12 August 2026, this Paper Citation Record lists 26 of 26 outbound references and 0 inbound Pith citation observations for arXiv:2503.04756."}