{"as_of":"2026-08-22T12:19:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:5ceb986becda47e94f858de137c4dbccef57676360de370226bc429ef2c4db7f","coverage":[{"denominator":47,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":47,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-15T20:30:13.206926Z","state":"measured"},{"denominator":48,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":48,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-22T06:32:14.747728+00:00","state":"measured"},{"denominator":1,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":1,"source":"paper_references, paper_reference_links","source_observed_at":"2026-05-18T10:04:39.223895Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-05-18T10:06:13.816265Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"cited_work":{"arxiv_id":"2505.12938","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2505.12938","snapshot_observed_at":"2026-06-05T21:23:00.469572Z","title":"Leveraging LLM Inconsistency to Boost Pass@ k Performance","venue":null,"work_id":"9cddb07d-6f2b-43a3-9b62-22fc1982a7d1","year":2025},"citing_paper":{"arxiv_id":"2510.04265","last_updated":"2026-05-12T01:55:07Z","snapshot_observed_at":"2026-08-13T07:36:21.999871Z","submitted_at":"2025-10-05T16:14:03Z","title":"Don't Pass@k: A Bayesian Framework for Large Language Model Evaluation","version":4},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-05-18T10:04:39.223895Z"},"links":{"cited_paper":"/paper/2505.12938","citing_paper":"/paper/2510.04265"},"observation_digest":"sha256:8cd6ce6f899d8ea931d8897979f49178025d419bbb60d674e4373dc495738333","observation_id":"dc005f4f-05d3-4139-a303-5adb886c2fb3","resolution":{"observed_at":"2026-05-18T10:06:13.819298Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2505.12938/citation-record","integrity":"/paper/2505.12938/integrity","json":"/paper/2505.12938/citation-record.json","paper":"/paper/2505.12938"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.934858Z","title":"The economic potential of generative AI: The next productivity frontier","venue":null,"work_id":"66c26887-6b5a-4903-a09a-ed828b3e677c","year":2023},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:12.957810Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:15ac2fcce95f6a489cb07fb5bc6dbca1a49206a89518e994f6ace340baf88a00","observation_id":"001811c3-c06e-4feb-a246-90a0cdfae81c","resolution":{"observed_at":"2026-08-15T20:30:13.940072Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.919716Z","title":"The language of prompting: What linguistic properties make a prompt successful?, 2023","venue":null,"work_id":"6ff38c57-0a25-4cf3-9f32-36cccbb235ca","year":2023},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:12.963512Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:7620c8045a7018b2d883f51f1db0e63bde981450182b2b3e6029c596ad4df1a6","observation_id":"14a3b913-0044-45f4-8b0a-6f8f49158bcd","resolution":{"observed_at":"2026-08-15T20:30:13.924468Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.904904Z","title":"Robustness of learning from task instructions, 2022","venue":null,"work_id":"aa4e0496-a67c-4cbb-8947-a1cc655c64ac","year":2022},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:12.969017Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:946a7731fd65aa111b7f63dd05251ae8aa595c545314aa96fcf0291e441d3bad","observation_id":"84dff12b-de9d-4e32-b417-cdac8815abee","resolution":{"observed_at":"2026-08-15T20:30:13.910041Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2308.11483","last_updated":"2023-08-22T14:54:59Z","snapshot_observed_at":"2026-08-19T22:01:31.805341Z","submitted_at":"2023-08-22T14:54:59Z","title":"Large Language Models Sensitivity to The Order of Options in Multiple-Choice Questions","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.11483","snapshot_observed_at":"2026-08-15T20:30:12.974774Z","title":"Large language models sensitivity to the order of options in multiple-choice questions, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:12.974774Z"},"links":{"cited_paper":"/paper/2308.11483","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:05ecf22c1caaf87745d0bfc59643e8965a13dc8b8f851d393c3ec8bc42aef530","observation_id":"049659ac-5db1-4a38-b987-a7c69bbbc802","resolution":{"observed_at":"2026-08-15T20:30:12.974774Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.889049Z","title":"When benchmarks are targets: Revealing the sensitivity of large language model leaderboards","venue":null,"work_id":"a8399bc7-19bb-4295-a131-8d9e35db5636","year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:12.980652Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:c178a9a80f77eab9038c9ad335a127d37e8529f492338ac0951393e63bba1e21","observation_id":"bc25770c-8471-4f25-a152-9430e640b13e","resolution":{"observed_at":"2026-08-15T20:30:13.894304Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.873691Z","title":"PertEval: Unveiling real knowledge capacity of LLMs with knowledge-invariant perturbations","venue":null,"work_id":"048568cd-af79-4883-8d1c-e8c45a3be754","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:12.986070Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:b20e43f251852d4512984e77432d1da665bd5199dab1657528ebb5f343b84893","observation_id":"4045c549-ac84-4aca-84fe-b4e25b88e8cc","resolution":{"observed_at":"2026-08-15T20:30:13.878317Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2410.12405","last_updated":"2024-10-16T09:38:13Z","snapshot_observed_at":"2026-08-20T00:22:48.174101Z","submitted_at":"2024-10-16T09:38:13Z","title":"ProSA: Assessing and Understanding the Prompt Sensitivity of LLMs","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.12405","snapshot_observed_at":"2026-08-15T20:30:12.997523Z","title":"Prosa: Assessing and understanding the prompt sensitivity of LLMs, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:12.997523Z"},"links":{"cited_paper":"/paper/2410.12405","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:5da1643798a3b1857c7fa0a663580ad20bdefaa3011b820e4db1bcc32e027398","observation_id":"b544a02a-374a-432a-8078-8752f5855fc1","resolution":{"observed_at":"2026-08-15T20:30:12.997523Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.859074Z","title":"Mind your format: Towards consistent evaluation of in-context learning improvements, 2024","venue":null,"work_id":"fb9eff04-eb71-474e-a0d9-7f5e2ae9e30c","year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.003997Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:d7f1015a271f74bbf91c070c95533db9d2e9b7b5ec8480265b68cc15c4cd3f0d","observation_id":"ae268e7d-e0cf-4f6f-9657-308e2f99f518","resolution":{"observed_at":"2026-08-15T20:30:13.863787Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.844967Z","title":"State of what art? A call for multi-prompt LLM evaluation","venue":null,"work_id":"783cb4e8-48c4-4459-a1de-bd04383b15e2","year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.009880Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:dc778d32bc8b17ac2cd28d34b4ce60a2bce459e38fefe9e5d519923e8b017d41","observation_id":"20ae5528-f764-4874-8795-a67ee1de55a7","resolution":{"observed_at":"2026-08-15T20:30:13.849170Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.829321Z","title":"Unnatural instructions: Tuning language models with (almost) no human labor, 2022","venue":null,"work_id":"8f077f39-dbd3-4a05-b79d-28bd1948f181","year":2022},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.015169Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:c0661fc2e5c52ef83114f439f428bcf50adee305f6a17054091bcd76dfac6f31","observation_id":"75fb61e9-5df3-42e7-bbdf-f77d73b08202","resolution":{"observed_at":"2026-08-15T20:30:13.834279Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2306.11270","last_updated":"2023-07-09T00:39:30Z","snapshot_observed_at":"2026-08-16T15:22:41.609607Z","submitted_at":"2023-06-20T03:48:51Z","title":"Evaluating the Zero-shot Robustness of Instruction-tuned Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2306.11270","snapshot_observed_at":"2026-08-15T20:30:13.021272Z","title":"Evaluating the zero-shot robustness of instruction-tuned language models, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.021272Z"},"links":{"cited_paper":"/paper/2306.11270","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:53c3e55880217374c0107abcaffcfc18dcc58832093acbc50842811e36b7c5bf","observation_id":"21ace7bd-d718-4556-80b4-7b331f75d834","resolution":{"observed_at":"2026-08-15T20:30:13.021272Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.11324","last_updated":"2024-07-01T22:28:01Z","snapshot_observed_at":"2026-08-12T15:05:30.306144Z","submitted_at":"2023-10-17T15:03:30Z","title":"Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.11324","snapshot_observed_at":"2026-08-15T20:30:13.026850Z","title":"Quantifying language models’ sensitivity to spurious features in prompt design, or: How I learned to start worrying about prompt formatting, 2023","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.026850Z"},"links":{"cited_paper":"/paper/2310.11324","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:afe48a1b86a78a33cf29343986f2825d882a398ba13e587fd95180bbc36df15c","observation_id":"6114d2c4-7047-4f56-b94b-82d5f168f982","resolution":{"observed_at":"2026-08-15T20:30:13.026850Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2107.03374","last_updated":"2021-07-14T17:16:02Z","snapshot_observed_at":"2026-08-20T20:50:23.483838Z","submitted_at":"2021-07-07T17:41:24Z","title":"Evaluating Large Language Models Trained on Code","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2107.03374","snapshot_observed_at":"2026-08-15T20:30:13.032426Z","title":null,"venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.032426Z"},"links":{"cited_paper":"/paper/2107.03374","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:1ca0ac8ca1f83f22f53f49760066048c008dec9ad61ef25f1b08555eb63f8ce3","observation_id":"26236707-aa42-4280-96b6-a4e75b7814e8","resolution":{"observed_at":"2026-08-15T20:30:13.032426Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.814079Z","title":"Spoc: Search-based pseudocode to code","venue":null,"work_id":"a0bec898-3608-4d39-9f04-eede053d853a","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.037920Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:0bba0bc4590ae0cf1bd823e57f519d57d3da33f8dade34eec44562fcb0ed3030","observation_id":"dfa58b10-3b05-4093-b2e4-078cb97018bb","resolution":{"observed_at":"2026-08-15T20:30:13.819052Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.782732Z","title":"AI-powered fuzzing: Breaking the bug hunting barrier, 2023","venue":null,"work_id":"00821dc6-b919-4412-a45f-b17c8c910f0b","year":2023},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.050494Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:e5cb7ae2cefa41f802b0ac004a51171ae11a60d981494506ada2a89bcd7c7b5a","observation_id":"40548b60-6a9a-4541-a324-1e52a2e3f06d","resolution":{"observed_at":"2026-08-15T20:30:13.787428Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2412.16720","last_updated":"2026-04-30T02:46:40Z","snapshot_observed_at":"2026-08-19T11:46:55.171293Z","submitted_at":"2024-12-21T18:04:31Z","title":"OpenAI o1 System Card","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2412.16720","snapshot_observed_at":"2026-08-15T20:30:13.055766Z","title":"OpenAI o1 system card, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.055766Z"},"links":{"cited_paper":"/paper/2412.16720","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:9ab69549f4b4dbbb50e2e4641139e5600d0e14d140a3d301b1ac6ad3eb0cd79b","observation_id":"a79e3131-e466-4cec-bb40-475eaf9586ed","resolution":{"observed_at":"2026-08-15T20:30:13.055766Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.767480Z","title":"OpenAI o3-mini system card, 2025","venue":null,"work_id":"c7a9d5ad-846b-4fd3-a025-303dcc51705e","year":2025},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.060874Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:9a798e098a2f57fd82927634619c85d42e3492c1b81dd0cb7c646054f67525bb","observation_id":"055843a1-3ca4-4375-98ab-2084d87348d7","resolution":{"observed_at":"2026-08-15T20:30:13.772214Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.752143Z","title":"Claude 3.7 Sonnet system card, 2025","venue":null,"work_id":"10123a52-1e19-44e0-9337-9a7c013c8356","year":2025},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.066359Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:6acb43551eebe95a349fa9f8fa9446fd481444d5e7f1e6af3bd3a2bba205e94f","observation_id":"b7b90270-1a70-4c12-b448-2cfb644155c7","resolution":{"observed_at":"2026-08-15T20:30:13.757034Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2104.08786","last_updated":"2022-03-03T12:10:58Z","snapshot_observed_at":"2026-08-18T15:42:46.886862Z","submitted_at":"2021-04-18T09:29:16Z","title":"Fantastically Ordered Prompts and Where to Find Them: Overcoming Few-Shot Prompt Order Sensitivity","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2104.08786","snapshot_observed_at":"2026-08-15T20:30:13.071631Z","title":"Fantastically ordered prompts and where to find them: Overcoming few-shot prompt order sensitivity, 2021","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.071631Z"},"links":{"cited_paper":"/paper/2104.08786","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:518899c345ae08d561da81b0131dc43630ee37357a6e12352d7409e8c856313f","observation_id":"64acb600-91b5-4841-b4e3-4981a1682fab","resolution":{"observed_at":"2026-08-15T20:30:13.071631Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2410.02185","last_updated":"2024-10-04T07:00:03Z","snapshot_observed_at":"2026-08-18T17:09:03.060687Z","submitted_at":"2024-10-03T04:01:14Z","title":"POSIX: A Prompt Sensitivity Index For Large Language Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2410.02185","snapshot_observed_at":"2026-08-15T20:30:13.077068Z","title":"Posix: A prompt sensitivity index for large language models, 2024","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.077068Z"},"links":{"cited_paper":"/paper/2410.02185","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:23e5e48fb9cfcb4feb49324b27e2188ca77c0006ad73f356aca6d3a10088480f","observation_id":"1025282a-beb9-40cd-904b-b52a063da18c","resolution":{"observed_at":"2026-08-15T20:30:13.077068Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.736555Z","title":"DOVE: A large-scale multi-dimensional predictions dataset towards meaningful LLM evaluation, 2025","venue":null,"work_id":"c6652b24-bd89-40e9-9723-8a8d1f3d0458","year":2025},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.082080Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:3edd08eb2c01ec904fc12d3d1fcb1f7fa68be7b9939b15b35c7950f038df1157","observation_id":"054d1558-6a52-4af8-9aa3-f9d6a35e9e8f","resolution":{"observed_at":"2026-08-15T20:30:13.742021Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2212.04037","last_updated":"2024-09-12T19:54:37Z","snapshot_observed_at":"2026-08-20T08:08:27.599725Z","submitted_at":"2022-12-08T02:21:47Z","title":"Demystifying Prompts in Language Models via Perplexity Estimation","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.04037","snapshot_observed_at":"2026-08-15T20:30:13.086775Z","title":"Demystifying prompts in language models via perplexity estimation, 2022","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.086775Z"},"links":{"cited_paper":"/paper/2212.04037","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:79ed99991f8870509d9e506fcf9d4619bad99187231d28c0e6d153f03959b51d","observation_id":"6d85e5b4-b88d-4886-af06-4796969e0924","resolution":{"observed_at":"2026-08-15T20:30:13.086775Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2105.09938","last_updated":"2021-11-08T21:16:44Z","snapshot_observed_at":"2026-08-14T15:19:05.440904Z","submitted_at":"2021-05-20T17:58:42Z","title":"Measuring Coding Challenge Competence With APPS","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2105.09938","snapshot_observed_at":"2026-08-15T20:30:13.091733Z","title":"Measuring coding challenge competence with APPS","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.091733Z"},"links":{"cited_paper":"/paper/2105.09938","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:46f315bf1ba97fa80b80b7eff31808c28c0ad5a4db6a3eeef5789b300aabaaaa","observation_id":"7db39a4e-a403-4208-9533-4ba61f1bc2c1","resolution":{"observed_at":"2026-08-15T20:30:13.091733Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.721700Z","title":"NYU CTF bench: A scalable open-source benchmark dataset for evaluating LLMs in offensive security","venue":null,"work_id":"4b3dec0b-05e2-4696-8221-b0d66c4485a0","year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.096697Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:a5311b35c8d156b2dcdef0ecc0fe4a667d06121f458a67b42bfca959261501ac","observation_id":"917100e3-ff85-415f-831c-f0d98fd59828","resolution":{"observed_at":"2026-08-15T20:30:13.726480Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.706904Z","title":"OpenAI o3 and o4-mini system card, 2025","venue":null,"work_id":"f06dfc9a-2fa8-46eb-871b-e12e7b7975f7","year":2025},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.101531Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:6654a9924410a3f23fcea4ee647799b9125bc4ff5311c6d91c11687b7ec88090","observation_id":"ebdbc089-1e15-4316-b6f6-18b6038ba51b","resolution":{"observed_at":"2026-08-15T20:30:13.711876Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2503.11917","last_updated":"2025-04-21T19:22:25Z","snapshot_observed_at":"2026-08-20T13:13:43.726248Z","submitted_at":"2025-03-14T23:05:02Z","title":"A Framework for Evaluating Emerging Cyberattack Capabilities of AI","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2503.11917","snapshot_observed_at":"2026-08-15T20:30:13.107210Z","title":"A framework for evaluating emerging cyberattack capabilities of ai","venue":null,"work_id":null,"year":2025},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.107210Z"},"links":{"cited_paper":"/paper/2503.11917","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:531088204cc599b8320d69aadbc0b7c455df1bb43a589c0f9686402acfbb18cc","observation_id":"c80dba00-874f-4d64-8dbd-0988ac42796d","resolution":{"observed_at":"2026-08-15T20:30:13.107210Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.692839Z","title":"Amazon Bedrock","venue":null,"work_id":"178c6b23-41f5-42bb-9fd9-73868d15993b","year":2023},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.112553Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:0db3dd28f274c8ea65211b64ff2735832ee4f2bc8827ca29727d1355f09d861f","observation_id":"9e313fbb-6ff4-4599-9e99-b8655b4cb042","resolution":{"observed_at":"2026-08-15T20:30:13.697516Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.678976Z","title":"Openai api","venue":null,"work_id":"bce6e4c2-3138-410b-8bc1-8fbf0729103f","year":2021},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.117645Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:404345928322df7f3e4cd6dd992f6d4dd6f197191b77c44b2a65a1b0b1be09f1","observation_id":"173b315b-979d-4164-814d-1bb3fe68309a","resolution":{"observed_at":"2026-08-15T20:30:13.683092Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.665434Z","title":"Azure OpenAI service","venue":null,"work_id":"5750e75c-d38f-4cb8-9f3e-7de00f344bda","year":2023},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.122743Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:eacd05636dd16279b8ea651e4729ea60186661cbc2a38c90b9695355669c4404","observation_id":"44a75d7f-4cc0-4f3a-be6a-3103d3e648ac","resolution":{"observed_at":"2026-08-15T20:30:13.669725Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.650444Z","title":"pickle — python object serialization, 2025","venue":null,"work_id":"21b0d98b-f27c-4b30-9e8c-a304d2c5a768","year":2025},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.132852Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:e2ec92c37138296f5e8a0d314c0b785ed11da89ba31784ac8873e6bd2948e7b3","observation_id":"8002fa64-242f-4fc9-97d0-25b4c2c7c079","resolution":{"observed_at":"2026-08-15T20:30:13.655720Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.636047Z","title":"t2.large","venue":null,"work_id":"aee753a3-d51c-4ae9-ab65-258e8c346b14","year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.137916Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:100ce0c77601c015949ddc48b310afb789b4b1242cc01ebb29ffbd1f25afddd7","observation_id":"a7716a26-4cf5-424e-9f0c-5ae892af3ffb","resolution":{"observed_at":"2026-08-15T20:30:13.640559Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.621791Z","title":null,"venue":null,"work_id":"d0696b88-2ef5-443c-91ce-aa71ec1c08be","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.143759Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:03a2ee175052f7edcafe768f04e6bf35aef88813c13d987adc5e25cbeedbbf26","observation_id":"bc4a04d9-a225-4bc6-81e7-18979a6a2e34","resolution":{"observed_at":"2026-08-15T20:30:13.626295Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.607985Z","title":null,"venue":null,"work_id":"80c35ef5-5d2f-4ec4-84e2-366fe3578126","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.149443Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:912937f27e82876ed12e02bf36b86e58fdedf627cfbba524ece2725cb90ea8fc","observation_id":"44338ff1-5d1d-400f-9f68-6eb96beb422d","resolution":{"observed_at":"2026-08-15T20:30:13.612306Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.592703Z","title":null,"venue":null,"work_id":"fc190b9e-c9d9-43b9-80db-d8533340ba81","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.155094Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:5af7c5c9234c5f33a7f56b14473cfc7a1d9cb3546c2118c159430e2964f6bdd9","observation_id":"9617a151-5a42-409d-b4fc-b5579a5c0c2d","resolution":{"observed_at":"2026-08-15T20:30:13.597551Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.577657Z","title":null,"venue":null,"work_id":"0d876c96-ae9d-4172-8d80-0c6709ea705b","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.161093Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:d425c2c0b4bf46ae21dec17064de2c7f21e943bfd7b79e199c36d61c080defd6","observation_id":"2efb0b7a-70ca-4c43-b77e-4d339918589f","resolution":{"observed_at":"2026-08-15T20:30:13.582236Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.563482Z","title":"In this appendix, we provide the instructions given to these experts","venue":null,"work_id":"c5cd51ed-ce18-4781-8f12-66c3b5302b69","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.165853Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:bfd4adc7413304d75958b60c20f34739c25924111521755672f82af95f64d100","observation_id":"5d2254f8-d1c3-43cf-8c45-a707ecfb012a","resolution":{"observed_at":"2026-08-15T20:30:13.567896Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.549316Z","title":null,"venue":null,"work_id":"d874da31-2add-426c-b940-34dedb5bdfd4","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.171314Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:76dae622792bc006e094a6629145706bb5d91ebaba6ee8de412a6f6a083774ba","observation_id":"53fb84d6-9d9a-4fc7-9487-349c5dacae30","resolution":{"observed_at":"2026-08-15T20:30:13.553625Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.534736Z","title":null,"venue":null,"work_id":"d66ea969-aae6-4d15-8112-23eedcc70dfe","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.175932Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:f7b7710fefa514318811ce1729275a3150d989ecd873f3c493690b57d8af12ae","observation_id":"adf67d3b-3de4-404e-95c2-efa27b169ac6","resolution":{"observed_at":"2026-08-15T20:30:13.539235Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.518920Z","title":"Translating","venue":null,"work_id":"b271b721-773d-40b1-8bc3-53ae17c2cd76","year":2024},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.180906Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:4a27aeaac2bdbff830d54bd34caf332db32e52c87d6e991aae6b44fb6f7018df","observation_id":"5b316919-b601-4fb0-99a6-3bbb72b51c03","resolution":{"observed_at":"2026-08-15T20:30:13.524147Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.502248Z","title":null,"venue":null,"work_id":"7faa5d47-3285-4490-bdb4-02fe0b350d58","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.185419Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:f50caf235186de2d09723bcbd45d05a245e8ae9b14c4693f2eb1a3152f635fbe","observation_id":"d36779ed-a6a7-42cc-8f58-efcc23fd7338","resolution":{"observed_at":"2026-08-15T20:30:13.507631Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.486077Z","title":") Let me solve this step by step","venue":null,"work_id":"e95e419d-075d-4de3-a379-e97ee83008aa","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.189854Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:235637f1645d62d91343489110fafcb07afe33a29cacb915db0aaef804e20a35","observation_id":"c4ba2d4d-cb9d-408f-9868-154dd03580e7","resolution":{"observed_at":"2026-08-15T20:30:13.490929Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.470854Z","title":null,"venue":null,"work_id":"f4cd0f76-f01f-4f79-8aed-2f09f9d81740","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.194781Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:871a05d1a64135b4cf315c87ae1405c60488833140362a590dd999673e2a128f","observation_id":"0192c96a-0268-4e19-ba52-77e741299a75","resolution":{"observed_at":"2026-08-15T20:30:13.475571Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.455958Z","title":null,"venue":null,"work_id":"d80fb835-6ff9-40bb-a85e-1607c9b36997","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.199143Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:f51a17516ab548d9a599e72b4a5f5bb9b66c77785caa511e695404fbb8e6b1ac","observation_id":"1c981aa7-b035-4c2b-b67b-37abf5c2020c","resolution":{"observed_at":"2026-08-15T20:30:13.460555Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.440741Z","title":"mph” to “leaves per day,","venue":null,"work_id":"2db9c1ae-4551-4e17-90cd-55e86416f158","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.203142Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:8e634606a4913641ab63afc3815fc7251e54d97562a458c426d815fa83088dda","observation_id":"dd5bee7b-a92a-4eb1-a301-f0289d7caba0","resolution":{"observed_at":"2026-08-15T20:30:13.445316Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.423105Z","title":"Sample the results multiple times to obtain an average success rate","venue":null,"work_id":"2e921c0d-8d62-4876-8949-e389a94d5ac1","year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.206926Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:efee32e73bace166ebff1c5256da0c3a1aed842eac2744bf01232c8477c5d9c0","observation_id":"3493161a-5832-472a-9359-769bbdbb89b4","resolution":{"observed_at":"2026-08-15T20:30:13.429803Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-15T20:30:13.798602Z","title":null,"venue":null,"work_id":"3c8643cb-1ad4-4494-9a68-71372c9f63aa","year":2019},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":2019,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:13.044383Z"},"links":{"citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:c47b588aca14b40fae97581a41e35d10dc2bbe27e7f030224280cdfc046c4b90","observation_id":"6a455ac9-2075-46a7-bf10-9e1f36cfe491","resolution":{"observed_at":"2026-08-15T20:30:13.803622Z","resolver_source":"raw_fallback","status":"unresolved"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-22T06:32:14.747728+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.19740","last_updated":"2024-10-18T06:57:08Z","snapshot_observed_at":"2026-08-16T13:47:54.356093Z","submitted_at":"2024-05-30T06:38:32Z","title":"PertEval: Unveiling Real Knowledge Capacity of LLMs with Knowledge-Invariant Perturbations","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2405.19740","snapshot_observed_at":"2026-08-15T20:30:12.991801Z","title":null,"venue":null,"work_id":null,"year":null},"citing_paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance","version":2},"reference_index":2024,"source":"pdf_text","source_observed_at":"2026-08-15T20:30:12.991801Z"},"links":{"cited_paper":"/paper/2405.19740","citing_paper":"/paper/2505.12938"},"observation_digest":"sha256:f43f86a92551e8725fd759f6e00ecf32695a476be303ec3ead87b8203711e351","observation_id":"739490a0-e31e-487f-a4dc-8c1d5677c20a","resolution":{"observed_at":"2026-08-15T20:30:12.991801Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"paper":{"arxiv_id":"2505.12938","last_updated":"2025-05-20T14:22:15Z","latest_version":2,"primary_category":"cs.LG","snapshot_observed_at":"2026-08-18T15:43:57.374312Z","submitted_at":"2025-05-19T10:22:04Z","title":"Leveraging LLM Inconsistency to Boost Pass@k Performance"},"reference_resolution":{"displayed":47,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":22,"verified_exact":0,"verified_fuzzy":25},"total_outbound_references":47},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-22T06:32:14.747728+00:00","source":"crossref"},{"observed_at":"2026-08-22T06:32:06.552537+00:00","source":"retraction_watch"}],"thesis":"As of 22 August 2026, this Paper Citation Record lists 47 of 47 outbound references and 1 inbound Pith citation observation for arXiv:2505.12938."}