{"as_of":"2026-08-13T04:24:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:c77761c4b6197449b936d52e58fdb6b530e74110573427318a788a9c4725dfec","coverage":[{"denominator":13,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":13,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-11T22:05:53.920212Z","state":"measured"},{"denominator":13,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":13,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-12T06:34:41.77262+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2412.03854/citation-record","integrity":"/paper/2412.03854/integrity","json":"/paper/2412.03854/citation-record.json","paper":"/paper/2412.03854"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2203.00803","last_updated":"2022-03-02T00:39:00Z","snapshot_observed_at":"2026-08-02T16:12:03.961566Z","submitted_at":"2022-03-02T00:39:00Z","title":"Code Smells in Machine Learning Systems","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2203.00803","snapshot_observed_at":"2026-08-11T22:05:53.894301Z","title":"Garbage in, garbage out","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.894301Z"},"links":{"cited_paper":"/paper/2203.00803","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:2ae897d1a3456c97e390c638d37301b3f2995ac67619d47b0c6663f0bb8203df","observation_id":"487f5e55-1aab-4db4-9201-dac7ec187504","resolution":{"observed_at":"2026-08-11T22:05:53.894301Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T22:05:54.184119Z","title":null,"venue":null,"work_id":"809b0ce7-1f85-420f-8c0c-4e7de45df8cc","year":2018},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.899071Z"},"links":{"citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:ad6af59f04c72d7e04f46ca3f3b9e6f4285c8e33937b8f9f66a4b38e98babf21","observation_id":"f0000210-c02d-475d-9cf3-0300a8d61e12","resolution":{"observed_at":"2026-08-11T22:05:54.188169Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2109.05257","last_updated":"2022-01-04T08:21:34Z","snapshot_observed_at":"2026-08-13T01:32:19.114518Z","submitted_at":"2021-09-11T11:14:04Z","title":"Towards a Rigorous Evaluation of Time-series Anomaly Detection","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.05257","snapshot_observed_at":"2026-08-11T22:05:53.903223Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.903223Z"},"links":{"cited_paper":"/paper/2109.05257","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:afeb733d92fa4fad233274249fe2678fe36be6442dd06e776e78f5b280ba1ffb","observation_id":"d14a39b1-92d4-429e-ada4-92c0d9ce4449","resolution":{"observed_at":"2026-08-11T22:05:53.903223Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1611.03530","last_updated":"2017-02-26T19:36:40Z","snapshot_observed_at":"2026-08-01T16:56:59.989486Z","submitted_at":"2016-11-10T22:02:36Z","title":"Understanding deep learning requires rethinking generalization","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1611.03530","snapshot_observed_at":"2026-08-11T22:05:53.920212Z","title":"reproducibility","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.920212Z"},"links":{"cited_paper":"/paper/1611.03530","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:9a8b26eb6c77d79d7e539c754e6e55b69622be8ab535fb717313fc02cd8150f8","observation_id":"904c5b88-49f9-4a7b-8d0e-8caa768551fe","resolution":{"observed_at":"2026-08-11T22:05:53.920212Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T22:05:53.911795Z","title":"Global Ecology and Biogeography , 17(2): 145–151","venue":null,"work_id":null,"year":1996},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2008,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.911795Z"},"links":{"citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:64c6fd5627fd4b06f65bb1e25afc596c0c14eeec548a269d8837b46cff7eb922","observation_id":"477e324a-8a9e-4079-88ed-2a9dc3723ae8","resolution":{"observed_at":"2026-08-11T22:05:53.911795Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.06716","last_updated":"2022-10-06T15:12:56Z","snapshot_observed_at":"2026-08-10T21:26:51.477697Z","submitted_at":"2021-09-14T14:28:51Z","title":"HPOBench: A Collection of Reproducible Multi-Fidelity Benchmark Problems for HPO","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.06716","snapshot_observed_at":"2026-08-11T22:05:53.884166Z","title":"Transac- tions of the Association for Computational Linguistics , 5: 471–486","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2017,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.884166Z"},"links":{"cited_paper":"/paper/2109.06716","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:69fe7677779ae6ec53e5c52db5abc10ec5c3937045a865e2997ef4b4cb3f74f5","observation_id":"3d0f96e5-ca8d-4961-a042-913c4bcd8a71","resolution":{"observed_at":"2026-08-11T22:05:53.884166Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1809.06705","last_updated":"2020-04-25T12:29:53Z","snapshot_observed_at":"2026-08-11T15:35:01.299037Z","submitted_at":"2018-09-18T13:33:45Z","title":"Is rotation forest the best classifier for problems with continuous features?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1809.06705","snapshot_observed_at":"2026-08-11T22:05:53.865703Z","title":"In 2018 44th Euromicro Conference on Software Engineer- ing and Advanced Applications (SEAA) , 50–59","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2018,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.865703Z"},"links":{"cited_paper":"/paper/1809.06705","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:8ed9bc794db9f1086017cb633fab873bb43c05af402e1b8a4265cec9eb914aa6","observation_id":"fe24bc3f-0c00-4cc9-8043-5030859a57a5","resolution":{"observed_at":"2026-08-11T22:05:53.865703Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2006.14244","last_updated":"2020-06-25T08:20:10Z","snapshot_observed_at":"2026-08-06T23:02:29.267725Z","submitted_at":"2020-06-25T08:20:10Z","title":"On the Replicability and Reproducibility of Deep Learning in Software Engineering","version":1},"cited_work":{"arxiv_id":"2006.14244","doi":null,"metadata_source":"pith","pith_arxiv_id":"2006.14244","snapshot_observed_at":"2026-08-11T22:05:54.059118Z","title":"On the Replicability and Reproducibility of Deep Learning in Software Engineering","venue":"cs.SE","work_id":"4978e2e5-0a03-443a-a8e4-2715532d6342","year":2020},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2020,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.907585Z"},"links":{"cited_paper":"/paper/2006.14244","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:898c4135453d7c5858caec285c24cdb5a08f9b24abe3a5f53cdceb284728f4a0","observation_id":"1363cf5d-13de-49c7-a885-552f6f5112bb","resolution":{"observed_at":"2026-08-11T22:05:54.067551Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T22:05:54.170503Z","title":"In Proceedings of the 45th In- ternational ACM SIGIR Conference on Research and De- velopment in Information Retrieval , SIGIR ’22, 2790–2800","venue":null,"work_id":"e7e1f585-c0a4-4c1b-acfe-c618604d595d","year":2009},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.916101Z"},"links":{"citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:3f6be348812baae990553b00a9049feb47781e6c96ff0afe9b3dc3fa15d67f55","observation_id":"1f2ed059-e799-4dcf-8620-08ee585627b9","resolution":{"observed_at":"2026-08-11T22:05:54.174905Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2102.03034","last_updated":"2021-10-26T03:04:45Z","snapshot_observed_at":"2026-08-09T18:29:42.542131Z","submitted_at":"2021-02-05T07:30:43Z","title":"Hyperparameter Optimization Is Deceiving Us, and How to Stop It","version":4},"cited_work":{"arxiv_id":"2102.03034","doi":null,"metadata_source":"pith","pith_arxiv_id":"2102.03034","snapshot_observed_at":"2026-08-11T22:05:54.144323Z","title":"Hyperparameter Optimization Is Deceiving Us, and How to Stop It","venue":"cs.LG","work_id":"2d01d551-bdc4-455b-abdf-8b13d4c24045","year":2021},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2023,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.871238Z"},"links":{"cited_paper":"/paper/2102.03034","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:0e36e6d8a8ed761ea9493dacb11705b07ee3ec169ae5df2105a06ca6df761cb4","observation_id":"20279a08-b301-40f1-ae1d-1b874c149817","resolution":{"observed_at":"2026-08-11T22:05:54.148294Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-11T22:05:54.196667Z","title":"In Proceedings of the 2nd ACM Conference on Reproducibility and Replicability, ACM REP ’24, 36–45","venue":null,"work_id":"7de37f8c-6108-477b-a91c-124d50731318","year":2019},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.875338Z"},"links":{"citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:840ae1a838cb17c32d8758d80ef98f1f9c974ef5d6675b3c1b44b8b78e6967d9","observation_id":"61ab5c37-316e-4b16-9ece-191a5c0d1500","resolution":{"observed_at":"2026-08-11T22:05:54.201139Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"1909.03004","last_updated":"2019-09-06T16:40:42Z","snapshot_observed_at":"2026-07-06T08:19:45.897686Z","submitted_at":"2019-09-06T16:40:42Z","title":"Show Your Work: Improved Reporting of Experimental Results","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1909.03004","snapshot_observed_at":"2026-08-11T22:05:53.879583Z","title":"Dror, R.; Baumer, G.; Bogomolov, M.; and Reichart, R","venue":null,"work_id":null,"year":1909},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2194,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.879583Z"},"links":{"cited_paper":"/paper/1909.03004","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:c1ad5acb8f45fcd84d1d5eba3dad260c645dafc2343ee1878dba15a16b527928","observation_id":"db1b821f-6747-4aad-a775-4d9f74fdbb89","resolution":{"observed_at":"2026-08-11T22:05:53.879583Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.02157","last_updated":"2021-12-29T04:47:28Z","snapshot_observed_at":"2026-08-05T17:23:32.008962Z","submitted_at":"2021-09-05T19:37:34Z","title":"Learning with Holographic Reduced Representations","version":2},"cited_work":{"arxiv_id":"2109.02157","doi":null,"metadata_source":"pith","pith_arxiv_id":"2109.02157","snapshot_observed_at":"2026-08-11T22:05:54.104606Z","title":"Learning with Holographic Reduced Representations","venue":"cs.AI","work_id":"4116c238-1a47-4d2a-a938-8b76b698aed6","year":2021},"citing_paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?","version":1},"reference_index":2932,"source":"pdf_text","source_observed_at":"2026-08-11T22:05:53.889169Z"},"links":{"cited_paper":"/paper/2109.02157","citing_paper":"/paper/2412.03854"},"observation_digest":"sha256:c9ca06cda1ba93359d894d2c6117e1532861bbc73228bcb324e0bd06c7663783","observation_id":"37c3d6e8-ec3c-437e-9613-5d1d819291bc","resolution":{"observed_at":"2026-08-11T22:05:54.108465Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-12T06:34:41.77262+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2412.03854","last_updated":"2024-12-05T04:04:39Z","latest_version":1,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-12T19:50:52.924861Z","submitted_at":"2024-12-05T04:04:39Z","title":"What Do Machine Learning Researchers Mean by \"Reproducible\"?"},"reference_resolution":{"displayed":13,"state_counts":{"malformed_identifier":1,"metadata_mismatch":1,"parse_uncertain":0,"unresolved":7,"verified_exact":2,"verified_fuzzy":2},"total_outbound_references":13},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-12T06:34:41.77262+00:00","source":"crossref"},{"observed_at":"2026-08-12T06:34:36.333875+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 13 of 13 outbound references and 0 inbound Pith citation observations for arXiv:2412.03854."}