{"as_of":"2026-08-19T19:18:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:202575702af5b809329247ceffba1def148cd7a77fb0e550308c692a4fbc6892","coverage":[{"denominator":35,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":35,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-05T05:30:12.142238Z","state":"measured"},{"denominator":36,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":36,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-19T06:32:44.657259+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-14T13:32:31.523845Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2509.09705","snapshot_observed_at":"2026-07-14T13:32:31.523845Z","title":"The non-determinism of small LLMs: Evidence of low answer consistency in repetition trials of standard multiple-choice benchmarks, 2025","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.10202","last_updated":"2026-08-08T06:29:00Z","snapshot_observed_at":"2026-08-15T02:38:54.736482Z","submitted_at":"2026-07-11T08:26:48Z","title":"When Counterbalancing Hides the Bias: Access-Conditioned Position Lock in Forced-Choice LLM Evaluation","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-07-14T13:32:31.523845Z"},"links":{"cited_paper":"/paper/2509.09705","citing_paper":"/paper/2607.10202"},"observation_digest":"sha256:a88e249a69de9538fe1b9269a731040644e479db902d050c1ab438cb07108447","observation_id":"2eea4e8a-cf4f-4f95-b579-7ef6b518771e","resolution":{"observed_at":"2026-07-14T13:32:31.523845Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2509.09705/citation-record","integrity":"/paper/2509.09705/integrity","json":"/paper/2509.09705/citation-record.json","paper":"/paper/2509.09705"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:11.720305Z","title":"URL: \" 'urlintro :=","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.720305Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:6b1b55d7eddca7cbc2736fbb2b6ad79564fdfdc652c20682656c884f18c815f3","observation_id":"207ec1f2-74c1-4806-99a3-75d9ccbbc67c","resolution":{"observed_at":"2026-08-05T05:30:11.720305Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:11.731015Z","title":"write newline","venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.731015Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:1dc35b04e9a8ee0b2b9c4a64a5ba813dac00a7d234de94e2f5de3c618d5714a1","observation_id":"e349950d-c763-487e-82da-d14146b15091","resolution":{"observed_at":"2026-08-05T05:30:11.731015Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.770362Z","title":null,"venue":null,"work_id":"6955f9ec-10cf-4020-adcf-8331baf36b9c","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.752300Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:df3509c86ab2e3fa6c50aa1e869561c035ba1c73225bbbae2eb480fd5c8a4d72","observation_id":"7ef62e0d-e755-43c7-9b4e-7911f0e99891","resolution":{"observed_at":"2026-08-05T05:30:13.775939Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2408.04667","last_updated":"2025-04-02T15:17:02Z","snapshot_observed_at":"2026-08-16T13:28:09.369179Z","submitted_at":"2024-08-06T16:43:35Z","title":"Non-Determinism of \"Deterministic\" LLM Settings","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.04667","snapshot_observed_at":"2026-08-05T05:30:11.765974Z","title":"deterministic","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.765974Z"},"links":{"cited_paper":"/paper/2408.04667","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:5357ecf4207ae3002e3f155d4fee2ba69eb19e5b82bf70327ffa44517b866d6a","observation_id":"27856e2b-641e-4404-9e61-9f9bd02c79c9","resolution":{"observed_at":"2026-08-05T05:30:11.765974Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.743928Z","title":null,"venue":null,"work_id":"0523ebe0-e58d-4bbb-8219-ae44c828a0b3","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.778515Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:42359cd55e5c74f02dc066f968b215e15eae9d539f2138c52bd7c3c1585a20be","observation_id":"1f9eb626-f268-44a2-8aea-3cf7b858a701","resolution":{"observed_at":"2026-08-05T05:30:13.753752Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2502.14425","last_updated":"2025-06-05T00:49:05Z","snapshot_observed_at":"2026-08-16T12:56:44.035834Z","submitted_at":"2025-02-20T10:23:27Z","title":"A Survey on Data Contamination for Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.14425","snapshot_observed_at":"2026-08-05T05:30:11.788526Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.788526Z"},"links":{"cited_paper":"/paper/2502.14425","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:a718ebff732f543aecfe5512b4edef4394b53e35c651713dbf73ce3105e9f2ff","observation_id":"c0cd5982-cabc-4996-b23a-0b0358a8ed3c","resolution":{"observed_at":"2026-08-05T05:30:11.788526Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.705295Z","title":null,"venue":null,"work_id":"c496d78d-b56f-4e8e-b37a-bc665105d35c","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.799447Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:110bb5738bf071d0a45a3bbaf349df51bd25746719f0c08a18068d438ddde829","observation_id":"fb17db78-f3c6-4598-9b8a-b653b4d6195f","resolution":{"observed_at":"2026-08-05T05:30:13.713303Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.19437","last_updated":"2025-02-18T17:26:38Z","snapshot_observed_at":"2026-08-18T18:18:37.449517Z","submitted_at":"2024-12-27T04:03:16Z","title":"DeepSeek-V3 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.19437","snapshot_observed_at":"2026-08-05T05:30:11.816722Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.816722Z"},"links":{"cited_paper":"/paper/2412.19437","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:b85105d0e3bc6d43d3808a1d4f7124ea24b49039945f9c54a710462770991094","observation_id":"aabe2cdf-f60e-4cdd-bb5f-a12c9db5e4c6","resolution":{"observed_at":"2026-08-05T05:30:11.816722Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-05T05:30:11.824513Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.824513Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:69df0fe173c7edb0041f944f2d84d0797538521735ecf817917dcbaf46905ee6","observation_id":"08c015af-f399-487c-8984-c97a56ef9c14","resolution":{"observed_at":"2026-08-05T05:30:11.824513Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04127","last_updated":"2025-01-10T14:31:21Z","snapshot_observed_at":"2026-08-18T22:14:47.638195Z","submitted_at":"2024-06-06T14:49:06Z","title":"Are We Done with MMLU?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04127","snapshot_observed_at":"2026-08-05T05:30:11.831165Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.831165Z"},"links":{"cited_paper":"/paper/2406.04127","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:d6cd5928b0cd98307270b4098ff51a2bd91a4a9f43a27a4b7cb67e7e6c325445","observation_id":"eee64261-ae94-4e30-b07d-679bc36d1498","resolution":{"observed_at":"2026-08-05T05:30:11.831165Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.668372Z","title":null,"venue":null,"work_id":"383cf43c-c961-426c-a7b1-665daa59422d","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.852164Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:baecd4dd0e34e757d47e9d38966d673f77c4ee0994c87e989c96450a0e19a5ec","observation_id":"f5a04dc6-4864-4011-ad56-8d226b71bb46","resolution":{"observed_at":"2026-08-05T05:30:13.673780Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2304.08247","last_updated":"2025-03-18T21:31:51Z","snapshot_observed_at":"2026-08-18T08:20:50.087566Z","submitted_at":"2023-04-14T11:28:08Z","title":"MedAlpaca -- An Open-Source Collection of Medical Conversational AI Models and Training Data","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2304.08247","snapshot_observed_at":"2026-08-05T05:30:11.872307Z","title":"Adams, Jens-Michalis Papaioannou, Paul Grundmann, Tom Oberhauser, Alexei Figueroa, Alexander Löser, Daniel Truhn, and Keno K","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.872307Z"},"links":{"cited_paper":"/paper/2304.08247","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:619a964d56e2342f0ad28bf6d4ee31223f8ec04d69e2ed6dcf41ceaf4cfdafe4","observation_id":"99ad80cc-4ac6-481b-984d-5dae7d32e6e7","resolution":{"observed_at":"2026-08-05T05:30:11.872307Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06825","last_updated":"2023-10-10T17:54:58Z","snapshot_observed_at":"2026-08-17T20:30:34.016254Z","submitted_at":"2023-10-10T17:54:58Z","title":"Mistral 7B","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06825","snapshot_observed_at":"2026-08-05T05:30:11.881873Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.881873Z"},"links":{"cited_paper":"/paper/2310.06825","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:39fa32eea2e1fc6d96425ed2b0f6bec3bd3bb43f21e2978ca47a5d14bb5e7ee1","observation_id":"db36a074-6b34-4a90-b967-71adeb2ff198","resolution":{"observed_at":"2026-08-05T05:30:11.881873Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2401.04088","last_updated":"2024-01-08T18:47:34Z","snapshot_observed_at":"2026-08-13T19:43:49.936776Z","submitted_at":"2024-01-08T18:47:34Z","title":"Mixtral of Experts","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2401.04088","snapshot_observed_at":"2026-08-05T05:30:11.889738Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.889738Z"},"links":{"cited_paper":"/paper/2401.04088","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:d3a92dffac2a1b011275d28b0d11210b0396d053326df938a32b406e92d0612d","observation_id":"90097737-fd3f-46fd-9844-9b87c0ac3cd9","resolution":{"observed_at":"2026-08-05T05:30:11.889738Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:11.912244Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":16,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.912244Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:d12a7eac8633cbb7cd7473d97d0b691c2143e14f4066725f4aa412852821c0f5","observation_id":"a4229807-4463-4e7d-8db4-dbf26670bf7b","resolution":{"observed_at":"2026-08-05T05:30:11.912244Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.10373","last_updated":"2024-07-17T09:34:00Z","snapshot_observed_at":"2026-08-16T14:18:08.335122Z","submitted_at":"2024-02-15T23:39:04Z","title":"BioMistral: A Collection of Open-Source Pretrained Large Language Models for Medical Domains","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.10373","snapshot_observed_at":"2026-08-05T05:30:11.929147Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":17,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.929147Z"},"links":{"cited_paper":"/paper/2402.10373","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:831f3de91ae84f86f36cad0ff423556076b4e1c7615976156610f80303c23c2b","observation_id":"ebe18297-a12e-4106-8898-141bca6a9562","resolution":{"observed_at":"2026-08-05T05:30:11.929147Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.00543","last_updated":"2024-11-30T17:29:08Z","snapshot_observed_at":"2026-08-14T18:39:29.502729Z","submitted_at":"2024-11-30T17:29:08Z","title":"Evaluating the Consistency of LLM Evaluators","version":1},"cited_work":{"arxiv_id":"2412.00543","doi":null,"metadata_source":"pith","pith_arxiv_id":"2412.00543","snapshot_observed_at":"2026-08-05T05:30:12.796894Z","title":"Evaluating the Consistency of LLM Evaluators","venue":"cs.CL","work_id":"24b4f452-4506-45b6-8a5c-9100378f041d","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":18,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.935278Z"},"links":{"cited_paper":"/paper/2412.00543","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:65a3280af2dddc4e1a97994bd51736fe377db18aaf253286b9a0ec44120b02c9","observation_id":"8c171ac4-f432-4ce3-9ba7-6edd5f4bac2f","resolution":{"observed_at":"2026-08-05T05:30:12.810095Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.597047Z","title":null,"venue":null,"work_id":"32091aae-1912-4759-9dc6-10e3d5304a10","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":19,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.948137Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:c9856f13fa4396a8d63b14a14c881c95c0b0016bb291f0096b92bdce8111317a","observation_id":"cfaf754b-7eb9-4a4a-b26c-d547803a0840","resolution":{"observed_at":"2026-08-05T05:30:13.608082Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.553693Z","title":null,"venue":null,"work_id":"9292fdb0-8f1e-4e61-986e-87f088a3b7f8","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":20,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.955337Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:e8dc4a6a9bf724ba513d2fc97e0457a6492d6578001045d83174583713a6fc9f","observation_id":"d92a01ec-a93a-42f0-b1ec-822f0ae917fc","resolution":{"observed_at":"2026-08-05T05:30:13.563492Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.05229","last_updated":"2025-08-27T16:24:39Z","snapshot_observed_at":"2026-08-16T08:54:56.543625Z","submitted_at":"2024-10-07T17:36:37Z","title":"GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.05229","snapshot_observed_at":"2026-08-05T05:30:11.961290Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":21,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.961290Z"},"links":{"cited_paper":"/paper/2410.05229","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:c3488443eeaff91e9eedf95689a0a5913d28453c83c02cb228c207f9b74a99d0","observation_id":"1d5af5e3-1471-418f-989f-4be0fc108edc","resolution":{"observed_at":"2026-08-05T05:30:11.961290Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.00137","last_updated":"2025-02-28T19:27:29Z","snapshot_observed_at":"2026-08-16T12:54:09.521599Z","submitted_at":"2025-02-28T19:27:29Z","title":"SCORE: Systematic COnsistency and Robustness Evaluation for Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.00137","snapshot_observed_at":"2026-08-05T05:30:11.968973Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":22,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.968973Z"},"links":{"cited_paper":"/paper/2503.00137","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:7cd2a119595d8340fa58b1ebe34d890214942606df2c7fc7234e33ee042d20d3","observation_id":"9fe68723-2e1e-42f6-8e7c-9d7111755257","resolution":{"observed_at":"2026-08-05T05:30:11.968973Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.16452","last_updated":"2023-11-28T03:16:12Z","snapshot_observed_at":"2026-08-16T14:39:48.082697Z","submitted_at":"2023-11-28T03:16:12Z","title":"Can Generalist Foundation Models Outcompete Special-Purpose Tuning? Case Study in Medicine","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.16452","snapshot_observed_at":"2026-08-05T05:30:11.982290Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":23,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.982290Z"},"links":{"cited_paper":"/paper/2311.16452","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:434212008bafff463d9f9dd7bbaaecd8928d0f66ee0c7ea4f6f280dad8c7b884","observation_id":"32e84c90-d21c-415e-9b3a-d8d4390ea1a3","resolution":{"observed_at":"2026-08-05T05:30:11.982290Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.509936Z","title":null,"venue":null,"work_id":"79d9ab98-58d2-4df4-a905-b1db3de154a8","year":2025},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":24,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:11.995696Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:3cee1a565d1cea9f657d2c890655dbee310dfdcf94931a90015468c266af23d7","observation_id":"eb17a5ab-3b25-4dd2-821e-13418371b0cd","resolution":{"observed_at":"2026-08-05T05:30:13.520034Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.467928Z","title":null,"venue":null,"work_id":"d883e7f7-c075-4e37-9c4a-eec117ea9053","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":25,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.004059Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:7d08274c7d50dfcadeb2fe909e3ac0aa8f0d9a9a5cd6f8d4050184241f988fce","observation_id":"64e6d07a-7fba-4cb0-b81f-c7b18967a902","resolution":{"observed_at":"2026-08-05T05:30:13.475849Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:12.013568Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":26,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.013568Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:5449069ef50525711d13df44a61058a2d1d5385bb45d8cfcd48dc3ec1444a122","observation_id":"330975ec-2d3a-41a7-af3e-9c04f131139b","resolution":{"observed_at":"2026-08-05T05:30:12.013568Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-08-17T18:50:07.059564Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15115","snapshot_observed_at":"2026-08-05T05:30:12.023639Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":27,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.023639Z"},"links":{"cited_paper":"/paper/2412.15115","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:416d2dc80bdf2dfa57e7c5b2db0f039718ece1500ddbc545e298bcc567b44d7a","observation_id":"cb92023f-dffa-4cf7-8e2a-a2661d56cf30","resolution":{"observed_at":"2026-08-05T05:30:12.023639Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.00699","last_updated":"2025-07-09T19:56:02Z","snapshot_observed_at":"2026-08-16T14:04:51.278555Z","submitted_at":"2024-03-31T14:32:02Z","title":"A Comprehensive Survey of Contamination Detection Methods in Large Language Models","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.00699","snapshot_observed_at":"2026-08-05T05:30:12.032640Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":28,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.032640Z"},"links":{"cited_paper":"/paper/2404.00699","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:0bcaabb6cc6b469d7db76f9dc5427642eac179e289be20273a4f38057e003bf1","observation_id":"cdc53206-30bd-46d4-bf55-97148b1c1409","resolution":{"observed_at":"2026-08-05T05:30:12.032640Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2305.09617","last_updated":"2023-05-16T17:11:29Z","snapshot_observed_at":"2026-08-16T14:30:19.311365Z","submitted_at":"2023-05-16T17:11:29Z","title":"Towards Expert-Level Medical Question Answering with Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.09617","snapshot_observed_at":"2026-08-05T05:30:12.044204Z","title":"Sara Mahdavi, Joelle Barral, Dale Webster, Greg S","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":29,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.044204Z"},"links":{"cited_paper":"/paper/2305.09617","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:5646bc42b04888979e484374fa7a40811d65d5a28e58046ba9198dbad610b038","observation_id":"df548de7-c07b-4da8-be3d-345a703d8947","resolution":{"observed_at":"2026-08-05T05:30:12.044204Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.10457","last_updated":"2024-07-15T06:12:17Z","snapshot_observed_at":"2026-08-16T20:20:59.328492Z","submitted_at":"2024-07-15T06:12:17Z","title":"The Good, The Bad, and The Greedy: Evaluation of LLMs Should Not Ignore Non-Determinism","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.10457","snapshot_observed_at":"2026-08-05T05:30:12.058244Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":30,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.058244Z"},"links":{"cited_paper":"/paper/2407.10457","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:39e27f96d7399fe8d132d8747653358948824dcde6bd90106573a32ef0eb9df7","observation_id":"c64a07ba-1aca-4d10-bbe2-35b3646355a2","resolution":{"observed_at":"2026-08-05T05:30:12.058244Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2302.13971","last_updated":"2023-02-27T17:11:15Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2023-02-27T17:11:15Z","title":"LLaMA: Open and Efficient Foundation Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2302.13971","snapshot_observed_at":"2026-08-05T05:30:12.076195Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":31,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.076195Z"},"links":{"cited_paper":"/paper/2302.13971","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:7e112962b2727e3bb2981c462bb6ba280719679de395324fd45e75cb156685ac","observation_id":"1e6d670a-6487-463e-99e5-4146d2022ec8","resolution":{"observed_at":"2026-08-05T05:30:12.076195Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:13.413704Z","title":null,"venue":null,"work_id":"2753e81d-14d4-4585-bef0-63dbfbe00174","year":1939},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":32,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.087544Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:b4a364ca8746b647c827094cadc05ca73e2e2140e6ee1f081de32b88bcc5e27f","observation_id":"0d3f1bbd-4f92-41df-8291-c3b58d7aa63a","resolution":{"observed_at":"2026-08-05T05:30:13.423747Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T05:30:12.108055Z","title":null,"venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":33,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.108055Z"},"links":{"citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:d369bb50c4ed9f1f908c513c30342a4bf684e0396b641e55553ac39cf8aa9a5a","observation_id":"25c0a7a5-d43d-4815-abdb-7269cfbfa7e7","resolution":{"observed_at":"2026-08-05T05:30:12.108055Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01349","last_updated":"2024-12-06T11:54:40Z","snapshot_observed_at":"2026-08-16T14:22:10.281659Z","submitted_at":"2024-02-02T12:07:00Z","title":"LLMs May Perform MCQA by Selecting the Least Incorrect Option","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01349","snapshot_observed_at":"2026-08-05T05:30:12.120281Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":34,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.120281Z"},"links":{"cited_paper":"/paper/2402.01349","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:ff9d70c6364b8ea65b2f200689a8fcc68305345019a23496c99cf99d5215410e","observation_id":"d91d2787-d1d4-4461-9d7f-e4334cdea54e","resolution":{"observed_at":"2026-08-05T05:30:12.120281Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.07872","last_updated":"2024-03-12T17:59:48Z","snapshot_observed_at":"2026-08-16T14:10:28.083217Z","submitted_at":"2024-03-12T17:59:48Z","title":"Rethinking Generative Large Language Model Evaluation for Semantic Comprehension","version":1},"cited_work":{"arxiv_id":"2403.07872","doi":null,"metadata_source":"pith","pith_arxiv_id":"2403.07872","snapshot_observed_at":"2026-08-05T05:30:12.357764Z","title":"Rethinking Generative Large Language Model Evaluation for Semantic Comprehension","venue":"cs.CL","work_id":"beb34fd8-8fae-48ec-bbb5-b2d6f0fc0447","year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":35,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.131461Z"},"links":{"cited_paper":"/paper/2403.07872","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:aed33a23b5ba659eb9e1d333793804d68f2b8cf99a5adb183418967d4cb9974c","observation_id":"022b1ac3-f3d7-4b9e-8f2a-2a5bb6ddf8ed","resolution":{"observed_at":"2026-08-05T05:30:12.376125Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-19T06:32:44.657259+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.04244","last_updated":"2024-06-06T16:41:39Z","snapshot_observed_at":"2026-08-17T14:15:23.870937Z","submitted_at":"2024-06-06T16:41:39Z","title":"Benchmark Data Contamination of Large Language Models: A Survey","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.04244","snapshot_observed_at":"2026-08-05T05:30:12.142238Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks","version":1},"reference_index":36,"source":"arxiv_source","source_observed_at":"2026-08-05T05:30:12.142238Z"},"links":{"cited_paper":"/paper/2406.04244","citing_paper":"/paper/2509.09705"},"observation_digest":"sha256:b759ab0c1dc6d1837edc0bf21fe9e528ae4711aed1235a1b9855db6895821f7c","observation_id":"04b02f54-93e5-4da2-97ae-62edbabf7138","resolution":{"observed_at":"2026-08-05T05:30:12.142238Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2509.09705","last_updated":"2025-09-05T17:31:14Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-18T15:07:05.826261Z","submitted_at":"2025-09-05T17:31:14Z","title":"The Non-Determinism of Small LLMs: Evidence of Low Answer Consistency in Repetition Trials of Standard Multiple-Choice Benchmarks"},"reference_resolution":{"displayed":35,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":33,"verified_exact":2,"verified_fuzzy":0},"total_outbound_references":35},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-19T06:32:44.657259+00:00","source":"crossref"},{"observed_at":"2026-08-19T06:32:39.956319+00:00","source":"retraction_watch"}],"thesis":"As of 19 August 2026, this Paper Citation Record lists 35 of 35 outbound references and 1 inbound Pith citation observation for arXiv:2509.09705."}