{"as_of":"2026-08-21T13:13:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:ea07891eebd95ee4740a901bb233ede16061df12c3f29807b76799c053999547","coverage":[{"denominator":47,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":47,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-02T13:46:10.247199Z","state":"measured"},{"denominator":47,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":47,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-21T06:32:19.484+00:00","state":"measured"},{"denominator":0,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":0,"source":"paper_references, paper_reference_links","source_observed_at":null,"state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"cited_works","source_observed_at":null,"state":"measured"}],"external_citation_measurements":[],"inbound":[],"links":{"evidence":"/evidence","html":"/paper/2607.22554/citation-record","integrity":"/paper/2607.22554/integrity","json":"/paper/2607.22554/citation-record.json","paper":"/paper/2607.22554"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2303.08774","last_updated":"2024-03-04T06:01:33Z","snapshot_observed_at":"2026-08-17T09:58:46.058102Z","submitted_at":"2023-03-15T17:15:04Z","title":"GPT-4 Technical Report","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.08774","snapshot_observed_at":"2026-08-02T13:46:06.913577Z","title":"Achiam, S","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:06.913577Z"},"links":{"cited_paper":"/paper/2303.08774","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:5e2e9eda731bae6430108c9763c35ba36298bc8e2235b2cfe28af8b7654809b8","observation_id":"10914893-6e48-421d-8505-a2ccc6fbdcb3","resolution":{"observed_at":"2026-08-02T13:46:06.913577Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:06.981595Z","title":"Alzahrani, H","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:06.981595Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:02afd6f6f3f3c2d7b20950ab5ccdda7c27c1f5b37aabb0ce3a8e6f8ef7e2632a","observation_id":"596eada7-4f2b-4972-8b9a-163f2522c0b4","resolution":{"observed_at":"2026-08-02T13:46:06.981595Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2005.14165","last_updated":"2020-07-22T19:47:17Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2020-05-28T17:29:03Z","title":"Language Models are Few-Shot Learners","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2005.14165","snapshot_observed_at":"2026-08-02T13:46:07.073994Z","title":null,"venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.073994Z"},"links":{"cited_paper":"/paper/2005.14165","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:571d3f5f26d36bc774421b7631028463cad04fb1d083dc66702eb5256435645d","observation_id":"b717b53f-5b5b-4b0d-915d-6614c39ef06a","resolution":{"observed_at":"2026-08-02T13:46:07.073994Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:07.169312Z","title":"Burnell, W","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.169312Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:d6cfcdbd7eafd663c885050a61e9d02ee7b75e6afa5500f1f25312a3b8e9221d","observation_id":"ce6424a8-e04f-4a2d-87e9-96493113ac65","resolution":{"observed_at":"2026-08-02T13:46:07.169312Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:07.237953Z","title":"Carlini and D","venue":null,"work_id":null,"year":2017},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.237953Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:75800cdc3035597601e2b3c76cf0be864c4888de94268c4eec0d20cd01d3c3ad","observation_id":"4f3e8c84-92fa-4261-a5a1-1d303965e839","resolution":{"observed_at":"2026-08-02T13:46:07.237953Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":"10.18653/v1/2025.emnlp-main.776","metadata_source":"openalex","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Cheng, W","venue":null,"work_id":"406d5584-e90d-4a7f-844b-5158bbe187bd","year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.329004Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:6797fdcf8e1187d469c47d99f2dbcf9e73c267d40a32b33c1318a538d8a47e4b","observation_id":"d2b9a763-c9ee-4d0b-92be-9ff4c9e18048","resolution":{"observed_at":"2026-08-02T13:48:30.686990Z","resolver_source":"doi","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-21T06:32:19.484+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:07.413422Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.413422Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:b5965585713c0832b57167522ac04c506dccf88170b776d2bb5934d793507860","observation_id":"27735564-e6ed-48e1-a759-572d2726ad83","resolution":{"observed_at":"2026-08-02T13:46:07.413422Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2204.02311","last_updated":"2022-10-05T06:02:24Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2022-04-05T16:11:45Z","title":"PaLM: Scaling Language Modeling with Pathways","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2204.02311","snapshot_observed_at":"2026-08-02T13:46:07.475799Z","title":"Chowdhery, S","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.475799Z"},"links":{"cited_paper":"/paper/2204.02311","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:65d0ee7b57c6b4cf3924d47185f3685306633c045cba99cc03a0e95fcc54ba1d","observation_id":"f75f8ed8-dbda-489a-9afd-a7c8969dd37e","resolution":{"observed_at":"2026-08-02T13:46:07.475799Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-14T02:43:01.480086Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-02T13:46:07.542054Z","title":"Cobbe, V","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.542054Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:71db8d83c03239f27c8f13acabed6fdd38cdb7369a9f326a2a3c22af45d4fb73","observation_id":"2f7ab7e7-a642-4352-a0ba-3bb180369ff7","resolution":{"observed_at":"2026-08-02T13:46:07.542054Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:07.607889Z","title":"Dekoninck, M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.607889Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:57a2b01aa20c4b3f94aeb13c81ddbc1ccc6a0fcdd60349d627a9053728426de8","observation_id":"e15cc938-0404-4057-a0ce-3e460d47cdd6","resolution":{"observed_at":"2026-08-02T13:46:07.607889Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1810.04805","last_updated":"2019-05-24T20:37:26Z","snapshot_observed_at":"2026-08-14T18:16:28.847993Z","submitted_at":"2018-10-11T00:50:01Z","title":"BERT: Pre-training of Deep Bidirectional Transformers for Language Understanding","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1810.04805","snapshot_observed_at":"2026-08-02T13:46:07.660124Z","title":"Devlin, M.-W","venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.660124Z"},"links":{"cited_paper":"/paper/1810.04805","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:9edd16a9e4d2502f76828a31aced68a6f5779574d0c10ae85bf9cac8d5ec91fb","observation_id":"8438c37c-3c5b-4570-a8e3-5b4c99f9686b","resolution":{"observed_at":"2026-08-02T13:46:07.660124Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:07.729513Z","title":"Elazar, N","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.729513Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:3bf31100973cd9361f22a695a82947e6ed38acacaf33aeb29a3f94e6a4a652bf","observation_id":"63130074-c28e-4cb6-b638-6a99935d06cb","resolution":{"observed_at":"2026-08-02T13:46:07.729513Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:07.790012Z","title":null,"venue":null,"work_id":null,"year":2019},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.790012Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:9e1975968a21c0f0776a11eadbb07f507d0224796d1d078c73c7381933c0d6cb","observation_id":"7ec13489-cdbb-4739-93ea-1fc06ef9c218","resolution":{"observed_at":"2026-08-02T13:46:07.790012Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.16100","last_updated":"2025-02-28T17:02:23Z","snapshot_observed_at":"2026-08-20T00:49:25.877850Z","submitted_at":"2024-12-20T17:42:25Z","title":"Logical Consistency of Large Language Models in Fact-checking","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.16100","snapshot_observed_at":"2026-08-02T13:46:07.839786Z","title":"Ghosh, S","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.839786Z"},"links":{"cited_paper":"/paper/2412.16100","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:44801fc9eea1d82f98f67511fcbaf88239af5b238b4363bdf3f3bad5e7e0c0bb","observation_id":"45883ebb-322f-4835-8f46-ae0ae11a5bde","resolution":{"observed_at":"2026-08-02T13:46:07.839786Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.08493","last_updated":"2024-02-21T22:02:26Z","snapshot_observed_at":"2026-08-17T02:58:13.312362Z","submitted_at":"2023-08-16T16:48:57Z","title":"Time Travel in LLMs: Tracing Data Contamination in Large Language Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.08493","snapshot_observed_at":"2026-08-02T13:46:07.903416Z","title":"Golchin and M","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.903416Z"},"links":{"cited_paper":"/paper/2308.08493","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:6937a4aaf0e978d685e2b41a00d80b64044c2012f4899ea1d7870a7caa362fc7","observation_id":"03de9f84-b2c6-493e-aa62-e3ce2dedda00","resolution":{"observed_at":"2026-08-02T13:46:07.903416Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.06233","last_updated":"2025-04-28T00:00:43Z","snapshot_observed_at":"2026-08-16T14:44:16.502574Z","submitted_at":"2023-11-10T18:48:58Z","title":"Data Contamination Quiz: A Tool to Detect and Estimate Contamination in Large Language Models","version":7},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.06233","snapshot_observed_at":"2026-08-02T13:46:07.961355Z","title":"Golchin and M","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:07.961355Z"},"links":{"cited_paper":"/paper/2311.06233","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:1f5de630bf5b817738e549af337f49abfad8f3bc2a6ace9fa58bd3dd593cf31f","observation_id":"77061f62-a06e-4d80-bd86-f1785eb60333","resolution":{"observed_at":"2026-08-02T13:46:07.961355Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1412.6572","last_updated":"2015-03-20T20:19:16Z","snapshot_observed_at":"2026-08-19T07:17:20.004918Z","submitted_at":"2014-12-20T01:17:12Z","title":"Explaining and Harnessing Adversarial Examples","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1412.6572","snapshot_observed_at":"2026-08-02T13:46:08.024564Z","title":null,"venue":null,"work_id":null,"year":2014},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.024564Z"},"links":{"cited_paper":"/paper/1412.6572","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:0cf539a3763d6963cb3e6f8ee055b2c4a7a32363ae6bdcd643f3b0bdc6ddf1b6","observation_id":"bd217cc6-ce1d-47e8-bc2f-da8adea0a031","resolution":{"observed_at":"2026-08-02T13:46:08.024564Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.21783","last_updated":"2024-11-23T23:27:33Z","snapshot_observed_at":"2026-08-13T17:20:44.002518Z","submitted_at":"2024-07-31T17:54:27Z","title":"The Llama 3 Herd of Models","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.21783","snapshot_observed_at":"2026-08-02T13:46:08.091732Z","title":"Grattafiori, A","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.091732Z"},"links":{"cited_paper":"/paper/2407.21783","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:79e448ba58bd7da22f2556cf5260ae4527bb93afd5c377b3159b6a8dd19d607a","observation_id":"88cf836e-e13c-461c-a86b-5b685da159d5","resolution":{"observed_at":"2026-08-02T13:46:08.091732Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2501.12948","last_updated":"2026-01-04T03:57:36Z","snapshot_observed_at":"2026-08-15T12:33:55.451951Z","submitted_at":"2025-01-22T15:19:35Z","title":"DeepSeek-R1: Incentivizing Reasoning Capability in LLMs via Reinforcement Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2501.12948","snapshot_observed_at":"2026-08-02T13:46:08.188766Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.188766Z"},"links":{"cited_paper":"/paper/2501.12948","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:4a414930071ece61d06b6e144fb2c0432fcbf3f363722daad8cd11b4932c85bc","observation_id":"71156f81-cd47-4aa2-8671-d2dbfa5f6667","resolution":{"observed_at":"2026-08-02T13:46:08.188766Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:08.272905Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.272905Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:5ca707d74249e9d4a6579503f6d38da40aa255a8b97894f7e9cb034b7de6b837","observation_id":"a307188b-73ca-4202-a9f2-5062074b491e","resolution":{"observed_at":"2026-08-02T13:46:08.272905Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2103.03874","last_updated":"2021-11-08T21:30:18Z","snapshot_observed_at":"2026-08-21T04:25:41.592030Z","submitted_at":"2021-03-05T18:59:39Z","title":"Measuring Mathematical Problem Solving With the MATH Dataset","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2103.03874","snapshot_observed_at":"2026-08-02T13:46:08.384334Z","title":"Hendrycks, C","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.384334Z"},"links":{"cited_paper":"/paper/2103.03874","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:076649f35ae84c391ac6d29d7f9609eac31ae2464a9bd4ac68d4048799caf9ba","observation_id":"c6d10879-6694-4b7d-8fa7-fe7a5584f7d2","resolution":{"observed_at":"2026-08-02T13:46:08.384334Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:08.447963Z","title":"Jiang, Y","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.447963Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:2ccc54777be7bda4f32e9eb8ab1661ef2814ab29d830643c2e68fa8d4f304006","observation_id":"48f6235d-2149-47cb-afba-c2ce1ccacc46","resolution":{"observed_at":"2026-08-02T13:46:08.447963Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2005.00700","last_updated":"2020-10-07T03:12:45Z","snapshot_observed_at":"2026-08-10T21:22:19.220442Z","submitted_at":"2020-05-02T04:42:14Z","title":"UnifiedQA: Crossing Format Boundaries With a Single QA System","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2005.00700","snapshot_observed_at":"2026-08-02T13:46:08.528174Z","title":"Khashabi, S","venue":null,"work_id":null,"year":2020},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.528174Z"},"links":{"cited_paper":"/paper/2005.00700","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:1af536e9d7ea8ea921fe41b2a10ade18a527aebeaa7abd5b00805a3007ac1f4d","observation_id":"8e1f02f3-d142-4be7-8be9-6d5ec32cfebb","resolution":{"observed_at":"2026-08-02T13:46:08.528174Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:08.592377Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.592377Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:b594afce3237c4fb954759f310342f1d2677189c28cc6ac3fca385fb3909873a","observation_id":"5cb68d0b-8058-43ab-b64d-3343b1fb4f96","resolution":{"observed_at":"2026-08-02T13:46:08.592377Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:08.684247Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.684247Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:956c3187fd240546d394ce42e0a96b64d4c3c34fed02637d292340a6b937b51c","observation_id":"86d0d7f3-5acf-4317-b775-995494556946","resolution":{"observed_at":"2026-08-02T13:46:08.684247Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2509.04013","last_updated":"2025-09-04T08:43:27Z","snapshot_observed_at":"2026-08-14T16:00:37.515648Z","submitted_at":"2025-09-04T08:43:27Z","title":"On Robustness and Reliability of Benchmark-Based Evaluation of LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2509.04013","snapshot_observed_at":"2026-08-02T13:46:08.844007Z","title":"Lunardi, V","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.844007Z"},"links":{"cited_paper":"/paper/2509.04013","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:e51e23eab95d3f772c66ee344d27045ba13009057414ef211fbc416477316742","observation_id":"b9caf502-53cd-465a-a0b2-1d1934c03339","resolution":{"observed_at":"2026-08-02T13:46:08.844007Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:08.920095Z","title":"Meier, J","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.920095Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:dc5704f5536de5ea16c90c7193fffc958a6cf63e16e2c9d5bac7035c944ef1c2","observation_id":"142f522a-1065-4fcf-9eb0-6359d5b97258","resolution":{"observed_at":"2026-08-02T13:46:08.920095Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.05229","last_updated":"2025-08-27T16:24:39Z","snapshot_observed_at":"2026-08-16T08:54:56.543625Z","submitted_at":"2024-10-07T17:36:37Z","title":"GSM-Symbolic: Understanding the Limitations of Mathematical Reasoning in Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.05229","snapshot_observed_at":"2026-08-02T13:46:09.001294Z","title":"Mirzadeh, K","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.001294Z"},"links":{"cited_paper":"/paper/2410.05229","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:4947f0c704ab6d6b422225dbf04c5ac8f1d3ec5e5c1be4fff4f85e9752d6dd42","observation_id":"ee9152c8-6f06-4730-8337-399b16bf8f74","resolution":{"observed_at":"2026-08-02T13:46:09.001294Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:09.055517Z","title":"Mizrahi, G","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.055517Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:a8997adeae4b07a641e78bda62d57aa9ff7aeaeca7f9511bfc46168d972df844","observation_id":"e57383b0-e86a-4ba8-965d-9ab2c244edd0","resolution":{"observed_at":"2026-08-02T13:46:09.055517Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2503.00137","last_updated":"2025-02-28T19:27:29Z","snapshot_observed_at":"2026-08-16T12:54:09.521599Z","submitted_at":"2025-02-28T19:27:29Z","title":"SCORE: Systematic COnsistency and Robustness Evaluation for Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.00137","snapshot_observed_at":"2026-08-02T13:46:09.135848Z","title":"Nalbandyan, R","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.135848Z"},"links":{"cited_paper":"/paper/2503.00137","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:050d812d831a1bf9a5a270014cde9fe2571bdff98e8dfa9b1551c3ee977f19f0","observation_id":"bdfa029c-8a03-42d6-9bc1-f9cb2f82fea3","resolution":{"observed_at":"2026-08-02T13:46:09.135848Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:09.189491Z","title":"Pezeshkpour and E","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.189491Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:437d49191e50bbce931c47cef828f4d889402753e72e27c79cb1e0741efa1ebc","observation_id":"3d209073-0d78-4e4a-9706-38b2acb7d5d8","resolution":{"observed_at":"2026-08-02T13:46:09.189491Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2405.17202","last_updated":"2024-10-31T03:26:21Z","snapshot_observed_at":"2026-08-16T13:49:02.254502Z","submitted_at":"2024-05-27T14:24:47Z","title":"Efficient multi-prompt evaluation of LLMs","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.17202","snapshot_observed_at":"2026-08-02T13:46:09.251987Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.251987Z"},"links":{"cited_paper":"/paper/2405.17202","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:e289be7d340d2da915717161acb1deb48406ee8ef8f8806f543d063d8b0e2a3f","observation_id":"63d0e612-7a57-4e37-ac73-d86f7f36e6f1","resolution":{"observed_at":"2026-08-02T13:46:09.251987Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2412.15115","last_updated":"2025-01-03T02:18:21Z","snapshot_observed_at":"2026-08-17T18:50:07.059564Z","submitted_at":"2024-12-19T17:56:09Z","title":"Qwen2.5 Technical Report","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.15115","snapshot_observed_at":"2026-08-02T13:46:09.314188Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.314188Z"},"links":{"cited_paper":"/paper/2412.15115","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:c3d6f2c4b42e7cfd2ed2811c925579a1952f41bde00ed573d2ff5ff7e5360cc7","observation_id":"d7627d3f-aa13-4067-bcf7-f25315bb8d17","resolution":{"observed_at":"2026-08-02T13:46:09.314188Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1910.10683","last_updated":"2023-09-19T15:14:48Z","snapshot_observed_at":"2026-08-14T16:16:21.567225Z","submitted_at":"2019-10-23T17:37:36Z","title":"Exploring the Limits of Transfer Learning with a Unified Text-to-Text Transformer","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1910.10683","snapshot_observed_at":"2026-08-02T13:46:09.399690Z","title":"Raffel, N","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.399690Z"},"links":{"cited_paper":"/paper/1910.10683","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:9165dcf5aab0b05a7755d6d54bd1af89506d8d3646c44e5c2edc9e3e16efaf54","observation_id":"1f3d187d-8d6c-4ee4-a253-926d23321023","resolution":{"observed_at":"2026-08-02T13:46:09.399690Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2502.15924","last_updated":"2025-02-21T20:41:37Z","snapshot_observed_at":"2026-08-18T00:54:51.460343Z","submitted_at":"2025-02-21T20:41:37Z","title":"Improving Consistency in Large Language Models through Chain of Guidance","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2502.15924","snapshot_observed_at":"2026-08-02T13:46:09.473023Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.473023Z"},"links":{"cited_paper":"/paper/2502.15924","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:6cf3d1ebe7419f1ce1bbf2e0aeeeb8595c6dce33e0efb8519430fd618b762efa","observation_id":"39d8d600-75b0-452f-ae02-e1d3973a07b1","resolution":{"observed_at":"2026-08-02T13:46:09.473023Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2005.04118","last_updated":"2020-05-08T15:48:31Z","snapshot_observed_at":"2026-08-18T07:59:11.409581Z","submitted_at":"2020-05-08T15:48:31Z","title":"Beyond Accuracy: Behavioral Testing of NLP models with CheckList","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2005.04118","snapshot_observed_at":"2026-08-02T13:46:09.525133Z","title":null,"venue":null,"work_id":null,"year":2005},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.525133Z"},"links":{"cited_paper":"/paper/2005.04118","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:2bf9c0aafaa956d380971f9e762ee979e9eb23f135f7437c062f27cdef60241b","observation_id":"48282414-8bb1-4cee-aeb7-6aafdb4a4c1f","resolution":{"observed_at":"2026-08-02T13:46:09.525133Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.11324","last_updated":"2024-07-01T22:28:01Z","snapshot_observed_at":"2026-08-12T15:05:30.306144Z","submitted_at":"2023-10-17T15:03:30Z","title":"Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.11324","snapshot_observed_at":"2026-08-02T13:46:09.599956Z","title":"Sclar, Y","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.599956Z"},"links":{"cited_paper":"/paper/2310.11324","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:58e5cc5c26354317b1554f5c471f56725189c89416e8cad5e67708c1e0ae3525","observation_id":"4b5a835c-c89d-4baa-bb83-3058e788e72b","resolution":{"observed_at":"2026-08-02T13:46:09.599956Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.03923","last_updated":"2024-11-06T13:54:08Z","snapshot_observed_at":"2026-08-16T13:02:26.531858Z","submitted_at":"2024-11-06T13:54:08Z","title":"Evaluation data contamination in LLMs: how do we measure it and (when) does it matter?","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.03923","snapshot_observed_at":"2026-08-02T13:46:09.669375Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.669375Z"},"links":{"cited_paper":"/paper/2411.03923","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:e748521a15d6b26328607b0ae96751fb68253f5730f1a8bd9c45320ab7e62378","observation_id":"e9a46123-ac97-4ffe-a532-9b4ac9c3f628","resolution":{"observed_at":"2026-08-02T13:46:09.669375Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1312.6199","last_updated":"2014-02-19T16:33:14Z","snapshot_observed_at":"2026-08-15T16:41:15.505782Z","submitted_at":"2013-12-21T03:36:08Z","title":"Intriguing properties of neural networks","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1312.6199","snapshot_observed_at":"2026-08-02T13:46:09.723900Z","title":"Szegedy, W","venue":null,"work_id":null,"year":2013},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.723900Z"},"links":{"cited_paper":"/paper/1312.6199","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:ace32eab23f8fee2bd866c183b05f5f716d62365fe32ff1af55fc188944e02e1","observation_id":"a80b08e6-75a1-4926-a977-c62b1f877d6d","resolution":{"observed_at":"2026-08-02T13:46:09.723900Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2201.11903","last_updated":"2023-01-10T23:07:57Z","snapshot_observed_at":"2026-08-13T07:04:41.220509Z","submitted_at":"2022-01-28T02:33:07Z","title":"Chain-of-Thought Prompting Elicits Reasoning in Large Language Models","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2201.11903","snapshot_observed_at":"2026-08-02T13:46:09.775824Z","title":null,"venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.775824Z"},"links":{"cited_paper":"/paper/2201.11903","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:1e8a56c50d7a409401aa99997e85e59ecb04b77b4158b220e7d342d30a13698b","observation_id":"c03b4b3e-f46e-4d6e-b014-7b5eda569423","resolution":{"observed_at":"2026-08-02T13:46:09.775824Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2411.04368","last_updated":"2024-11-07T01:58:42Z","snapshot_observed_at":"2026-08-17T10:08:57.374438Z","submitted_at":"2024-11-07T01:58:42Z","title":"Measuring short-form factuality in large language models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2411.04368","snapshot_observed_at":"2026-08-02T13:46:09.835881Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.835881Z"},"links":{"cited_paper":"/paper/2411.04368","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:d1abc4abeca90cc01737ec8b5a463a66c8d63be09ee27aa793344432fecc0448","observation_id":"dfa52227-d9e8-44d8-83c7-ca29fde41da8","resolution":{"observed_at":"2026-08-02T13:46:09.835881Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2505.09388","last_updated":"2025-05-14T13:41:34Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2025-05-14T13:41:34Z","title":"Qwen3 Technical Report","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2505.09388","snapshot_observed_at":"2026-08-02T13:46:09.897940Z","title":null,"venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.897940Z"},"links":{"cited_paper":"/paper/2505.09388","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:90580c76056c8872ecbbf88242159a27589373cbb898ac7ed27b865512689dcc","observation_id":"ee3b26e5-b154-4bb5-a587-369a4003a05a","resolution":{"observed_at":"2026-08-02T13:46:09.897940Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2404.11500","last_updated":"2024-04-17T15:53:49Z","snapshot_observed_at":"2026-08-19T18:41:59.543987Z","submitted_at":"2024-04-17T15:53:49Z","title":"Paraphrase and Solve: Exploring and Exploiting the Impact of Surface Form on Mathematical Reasoning in Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.11500","snapshot_observed_at":"2026-08-02T13:46:09.953784Z","title":"ask the exact same thing","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:09.953784Z"},"links":{"cited_paper":"/paper/2404.11500","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:7c9de9aacf194cbde0f3020de5e5c3c8475a8e4465d92a3c96a0c9762372eaaf","observation_id":"82aee369-6288-4d86-8fb9-fd435c89823d","resolution":{"observed_at":"2026-08-02T13:46:09.953784Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:10.012221Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:10.012221Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:87a21b201ec09eedef2578ed621fb84e6e3787303f3e4ecfcc89f971604f28de","observation_id":"ab95698f-49c7-42ce-9d2d-7c085669a96c","resolution":{"observed_at":"2026-08-02T13:46:10.012221Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:10.107243Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:10.107243Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:a737d105998709a3a99c460aab8f564cba86137e69667c737f482c4334fa33f9","observation_id":"35f4f713-cf7f-4e13-9fbe-200e975f2b6e","resolution":{"observed_at":"2026-08-02T13:46:10.107243Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":null,"snapshot_observed_at":"2026-08-02T13:46:10.247199Z","title":"any-correct","venue":null,"work_id":null,"year":1985},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:10.247199Z"},"links":{"citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:35b74eb0c5fc7d0d09b65de0e7ab4961ddcc7ef342c05f5c94041b4af183a6f6","observation_id":"f37bc3e1-d55c-4f1d-bbdf-f919d32e6830","resolution":{"observed_at":"2026-08-02T13:46:10.247199Z","resolver_source":null,"status":"malformed_identifier"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2109.07958","last_updated":"2022-05-08T02:43:02Z","snapshot_observed_at":"2026-08-03T16:26:48.747700Z","submitted_at":"2021-09-08T17:15:27Z","title":"TruthfulQA: Measuring How Models Mimic Human Falsehoods","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2109.07958","snapshot_observed_at":"2026-08-02T13:46:08.764054Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy","version":1},"reference_index":2022,"source":"pdf_text","source_observed_at":"2026-08-02T13:46:08.764054Z"},"links":{"cited_paper":"/paper/2109.07958","citing_paper":"/paper/2607.22554"},"observation_digest":"sha256:7829431a5fe01dcb810e95b2b2c8dcbe2da47d3d20e586ed4664af4b4f4e83f2","observation_id":"9b58037e-10e7-4ba6-83cb-fbfcc93841f3","resolution":{"observed_at":"2026-08-02T13:46:08.764054Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2607.22554","last_updated":"2026-05-18T16:45:13Z","latest_version":1,"primary_category":"cs.AI","snapshot_observed_at":"2026-08-16T01:45:48.511937Z","submitted_at":"2026-05-18T16:45:13Z","title":"Same Question, Different Answers: Evaluating LLM Reliability Beyond Accuracy"},"reference_resolution":{"displayed":47,"state_counts":{"malformed_identifier":2,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":44,"verified_exact":1,"verified_fuzzy":0},"total_outbound_references":47},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-21T06:32:19.484+00:00","source":"crossref"},{"observed_at":"2026-08-21T06:32:16.066871+00:00","source":"retraction_watch"}],"thesis":"As of 21 August 2026, this Paper Citation Record lists 47 of 47 outbound references and 0 inbound Pith citation observations for arXiv:2607.22554."}