{"as_of":"2026-08-15T16:19:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:765300305c01378ef11ac43ff6f55939dafa2da25bd5faf66e042b310b35c197","coverage":[{"denominator":15,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":15,"source":"paper_references, paper_reference_links","source_observed_at":"2026-07-04T21:40:04.594380Z","state":"measured"},{"denominator":18,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":18,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-15T06:32:42.880941+00:00","state":"measured"},{"denominator":3,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":3,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-14T04:20:34.128103Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"pith","source_observed_at":"2026-08-11T20:08:14.591497Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2605.28840","snapshot_observed_at":"2026-07-12T06:28:52.906906Z","title":"arXiv preprint arXiv:2605.28840 (2026)","venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2607.02873","last_updated":"2026-07-03T02:20:04Z","snapshot_observed_at":"2026-08-13T12:23:39.241047Z","submitted_at":"2026-07-03T02:20:04Z","title":"Determinants and Limits of LLM Security-Tool Orchestration: A Study with HexStrike-AI","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-07-12T06:28:52.906906Z"},"links":{"cited_paper":"/paper/2605.28840","citing_paper":"/paper/2607.02873"},"observation_digest":"sha256:9bba4998f3817ede7b1486c3150bde52deb617b4a83caa4449d878b4e60c62bf","observation_id":"1d6a2fe6-0b4f-4d8c-b9ae-2b04ab3f8272","resolution":{"observed_at":"2026-07-12T06:28:52.906906Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"cited_work":{"arxiv_id":"2605.28840","doi":null,"metadata_source":"pith","pith_arxiv_id":"2605.28840","snapshot_observed_at":"2026-08-11T20:08:14.591497Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","venue":"cs.CL","work_id":"7e4571c8-bb59-4ce8-ac67-ff4710b97278","year":2026},"citing_paper":{"arxiv_id":"2608.09290","last_updated":"2026-08-11T03:48:22Z","snapshot_observed_at":"2026-08-15T09:12:36.690660Z","submitted_at":"2026-08-10T08:43:16Z","title":"OpenCodeReview: Determinism over Non-Determinism for Cost-Effective Agent-Based Code Review","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-11T20:08:14.363485Z"},"links":{"cited_paper":"/paper/2605.28840","citing_paper":"/paper/2608.09290"},"observation_digest":"sha256:21b88696a429973ca09987e11827fd3dc4f748b86dae6fb8090688aea6871880","observation_id":"8291d6d5-5a89-4c07-b573-664cdbabcc8f","resolution":{"observed_at":"2026-08-11T20:08:14.595449Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2605.28840","snapshot_observed_at":"2026-08-14T04:20:34.128103Z","title":null,"venue":null,"work_id":null,"year":2026},"citing_paper":{"arxiv_id":"2608.09290","last_updated":"2026-08-11T03:48:22Z","snapshot_observed_at":"2026-08-15T09:12:36.690660Z","submitted_at":"2026-08-10T08:43:16Z","title":"OpenCodeReview: Determinism over Non-Determinism for Cost-Effective Agent-Based Code Review","version":2},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-14T04:20:34.128103Z"},"links":{"cited_paper":"/paper/2605.28840","citing_paper":"/paper/2608.09290"},"observation_digest":"sha256:836dd167d3546d17d2b5ea9853ca15bbbf52dcfc282a3eeefc71930eddc2b32b","observation_id":"2cae528b-4854-46f2-851b-fdaec5cea5f4","resolution":{"observed_at":"2026-08-14T04:20:34.128103Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}}],"links":{"evidence":"/evidence","html":"/paper/2605.28840/citation-record","integrity":"/paper/2605.28840/integrity","json":"/paper/2605.28840/citation-record.json","paper":"/paper/2605.28840"},"outbound":[{"citation":{"cited_paper":{"arxiv_id":"2407.01502","last_updated":"2024-07-01T17:48:14Z","snapshot_observed_at":"2026-08-15T11:30:55.657973Z","submitted_at":"2024-07-01T17:48:14Z","title":"AI Agents That Matter","version":1},"cited_work":{"arxiv_id":"2407.01502","doi":"10.48550/arxiv.2407.01502","metadata_source":"pith","pith_arxiv_id":"2407.01502","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Siegel, Nitya Nadgir, and Arvind Narayanan","venue":"cs.LG","work_id":"07877e57-5393-47ee-ae5f-563ff8f9a6b2","year":2024},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":1,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"cited_paper":"/paper/2407.01502","citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:6d2b887a762fd1678f77935cda01181eeed02071d8536d5714af94d24a560e92","observation_id":"2820413a-b0d2-471e-b7ea-cae3e27d243a","resolution":{"observed_at":"2026-07-04T21:40:08.959105Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-04T01:08:07.572928+00:00","source":"crossref_status_cache"},{"observed_at":"2026-08-04T01:08:07.572928+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.969488Z","title":"Simple and Scalable Predictive Uncertainty Estimation Using Deep Ensembles","venue":null,"work_id":"3660e74a-9c13-4227-8198-9b90e10d43ec","year":2017},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":2,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:e5c33a7aaa37da3bcd3056cb6700b7da7abe92d581aa369e296dc3d507fab20c","observation_id":"24bd9db3-4bf6-4676-8d65-159c7b04e6ae","resolution":{"observed_at":"2026-07-04T21:40:08.971067Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.978825Z","title":"API-Bank : A Comprehensive Benchmark for Tool-Augmented LLMs","venue":null,"work_id":"3d1783c0-9640-4811-a5fe-50b721fb0996","year":2023},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":3,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:6efcd58dc4f9e1f445b409695145e5fdcc9f0cd737aca41edfc77654dac92ae2","observation_id":"c4923d33-4936-449a-ba5f-50e2e39040cf","resolution":{"observed_at":"2026-07-04T21:40:08.980586Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.976640Z","title":"AgentBench : Evaluating LLMs as Agents","venue":null,"work_id":"62c1f8cd-584d-4474-9362-b89837f96d05","year":2024},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":4,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:7056cdbb7b923c9b7a7e716242f046409c50a2e856e6ce6239d06d007ffd6072","observation_id":"29f4bf33-5b75-4cae-a1b0-dccee256f807","resolution":{"observed_at":"2026-07-04T21:40:08.978307Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.983617Z","title":"Fantastically Ordered Prompts and Where to Find Them: Overcoming Few-Shot Prompt Order Sensitivity","venue":null,"work_id":"85741905-339f-4342-ad66-9dadb0e2c8c1","year":2022},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":5,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:54c815081144d058a6691e0fe06c563a6f1f83e004ea2f19a564460c8e7874ca","observation_id":"8826c55f-c90b-4d34-afb6-e4ed7f29e5c4","resolution":{"observed_at":"2026-07-04T21:40:08.985460Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2602.11619","last_updated":"2026-07-15T00:10:32Z","snapshot_observed_at":"2026-08-14T14:29:33.599733Z","submitted_at":"2026-02-12T06:15:14Z","title":"When Agents Disagree With Themselves: Behavioral Consistency as an Uncertainty Signal for LLM Agents","version":2},"cited_work":{"arxiv_id":"2602.11619","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2602.11619","snapshot_observed_at":"2026-07-16T01:21:45.045605Z","title":"When agents disagree with themselves: Measuring behavioral consistency in LLM -based agents","venue":null,"work_id":"cf82032c-fa92-4ac3-b6fd-334efd0c0f72","year":2026},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":6,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"cited_paper":"/paper/2602.11619","citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:e4f0173223125dc2015d1c80807f31e661d7ca3481282026c24eca2ce2f9885b","observation_id":"58ea8c18-cbd7-4fe5-8569-fcf3b5f5f337","resolution":{"observed_at":"2026-07-16T01:21:45.045605Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.15334","last_updated":"2023-05-24T16:48:11Z","snapshot_observed_at":"2026-08-14T14:07:14.900729Z","submitted_at":"2023-05-24T16:48:11Z","title":"Gorilla: Large Language Model Connected with Massive APIs","version":1},"cited_work":{"arxiv_id":"2305.15334","doi":"10.48550/arxiv.2305.15334","metadata_source":"pith","pith_arxiv_id":"2305.15334","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Gorilla: Large Language Model Connected with Massive APIs","venue":"cs.CL","work_id":"126a464a-4a73-495f-b669-de1e44aa8f09","year":2023},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":7,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"cited_paper":"/paper/2305.15334","citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:9d109c3dab0b205e566d8db32eb94b828c17ad72da849e5ccbe1317be93a004d","observation_id":"5cc656bf-e0b3-437f-8d11-29abf3cd8675","resolution":{"observed_at":"2026-07-04T21:40:08.951739Z","resolver_source":"local_arxiv","status":"metadata_mismatch"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.974454Z","title":"True Few-Shot Learning with Language Models","venue":null,"work_id":"72e9a4ad-5de6-4448-8008-3d67a3538bc6","year":2022},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":8,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:b341db2432056830ecfd253a1034da3a9c30eee45fcd0fe77cad0ae3e50b4ca1","observation_id":"a0b67295-ebb5-4149-a60b-5683717d0fd4","resolution":{"observed_at":"2026-07-04T21:40:08.975849Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.986262Z","title":"ToolLLM : Facilitating Large Language Models to Master 16000+ Real-World APIs","venue":null,"work_id":"f2a18e0c-38ac-4872-922e-8972f24a1c1f","year":2024},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":9,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:37fe925d9cdd323a3c70e1d29e3ffc72617b6e94da0a68d73ef82e46424ef13e","observation_id":"e3e80ddd-7a48-40a1-8a4f-eff821a61795","resolution":{"observed_at":"2026-07-04T21:40:08.988145Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2405.06682","last_updated":"2024-10-16T23:19:46Z","snapshot_observed_at":"2026-08-13T00:14:28.486590Z","submitted_at":"2024-05-05T18:56:46Z","title":"Self-Reflection in LLM Agents: Effects on Problem-Solving Performance","version":3},"cited_work":{"arxiv_id":"2405.06682","doi":"10.48550/arxiv.2405.06682","metadata_source":"arxiv_reference","pith_arxiv_id":"2405.06682","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Self-reflection in llm agents: Effects on problem-solving performance","venue":"arXiv (Cornell University)","work_id":"4a6935ff-3a8a-4c88-be48-81b34dfdc2bc","year":2024},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":10,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"cited_paper":"/paper/2405.06682","citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:740165cf00a0312a3322ddebfcbe6adc67cce752029ae4c93796fa4910085eca","observation_id":"68a76fa4-6aeb-4cb1-9f99-57a638fb2337","resolution":{"observed_at":"2026-07-04T21:40:08.954429Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.981272Z","title":"Toolformer : Language Models Can Teach Themselves to Use Tools","venue":null,"work_id":"321eaccb-defb-4a5a-8dbf-e03014612948","year":2023},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":11,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:a80160cffe875e7e010cb374d65a3c5651b24b5aff3d629d8301e01fea9d155b","observation_id":"7c2c9479-1b95-4a89-be6f-3fc238a75c18","resolution":{"observed_at":"2026-07-04T21:40:08.982920Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.11324","last_updated":"2024-07-01T22:28:01Z","snapshot_observed_at":"2026-08-12T15:05:30.306144Z","submitted_at":"2023-10-17T15:03:30Z","title":"Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting","version":2},"cited_work":{"arxiv_id":"2310.11324","doi":"10.1109/cdc57313.2025.11312997","metadata_source":"pith","pith_arxiv_id":"2310.11324","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Quantifying Language Models' Sensitivity to Spurious Features in Prompt Design or: How I learned to start worrying about prompt formatting","venue":"cs.CL","work_id":"c3e484df-e654-4146-8c4e-a16ca850afc7","year":2023},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":12,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"cited_paper":"/paper/2310.11324","citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:9694b4af3bce2defc67ac382b994350c5321773ddd8e9f13b72be57b890d13ca","observation_id":"b750e918-9a7b-430d-a5cb-c5eef6908d31","resolution":{"observed_at":"2026-07-04T21:40:08.966556Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.981279Z","title":"Self-Consistency Improves Chain of Thought Reasoning in Language Models","venue":null,"work_id":"3863e4dd-139c-48a9-85d3-8c2375e14535","year":2023},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":13,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:d8c2a38d9004d32b0a5ac9377793b159b66038c5e470b0b16e06b512041eb7fc","observation_id":"50b41652-8c61-4d32-8590-7a1624e5e315","resolution":{"observed_at":"2026-07-04T21:40:08.983094Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2112.04359","last_updated":"2021-12-08T16:09:48Z","snapshot_observed_at":"2026-08-09T15:17:43.394064Z","submitted_at":"2021-12-08T16:09:48Z","title":"Ethical and social risks of harm from Language Models","version":1},"cited_work":{"arxiv_id":"2112.04359","doi":"10.48550/arxiv.2112.04359","metadata_source":"pith","pith_arxiv_id":"2112.04359","snapshot_observed_at":"2026-08-05T02:28:24.338817Z","title":"Ethical and social risks of harm from Language Models","venue":"cs.CL","work_id":"b4ce1c45-ef69-445a-a872-dbb785b485e9","year":2021},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":14,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"cited_paper":"/paper/2112.04359","citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:5c902fd07f2507cff3dc4c210eea93f09ebed4a65098c489e235a572c8a395f4","observation_id":"8da8965d-35cb-4628-a274-be211553ffdf","resolution":{"observed_at":"2026-07-04T21:40:08.955390Z","resolver_source":"local_arxiv","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-07-12T23:50:40.702477+00:00","source":"crossref_status_cache"},{"observed_at":"2026-07-12T23:50:40.702477+00:00","source":"openalex_status_cache"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-07-04T21:40:08.979033Z","title":"ReAct : Synergizing Reasoning and Acting in Language Models","venue":null,"work_id":"65614f51-5281-4633-812c-97df3900f5af","year":2023},"citing_paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines","version":1},"reference_index":15,"source":"arxiv_source","source_observed_at":"2026-07-04T21:40:04.594380Z"},"links":{"citing_paper":"/paper/2605.28840"},"observation_digest":"sha256:180d20ca6ed474e6935946907d4f713f56b2777cb0c300c0fef113f026376a7e","observation_id":"d33d085a-1893-442d-946c-b2933f12688c","resolution":{"observed_at":"2026-07-04T21:40:08.980637Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-15T06:32:42.880941+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2605.28840","last_updated":"2026-04-23T16:06:03Z","latest_version":1,"primary_category":"cs.CL","snapshot_observed_at":"2026-08-07T14:35:09.129059Z","submitted_at":"2026-04-23T16:06:03Z","title":"How Consistent Are LLM Agents? Measuring Behavioral Reproducibility in Multi-Step Tool-Calling Pipelines"},"reference_resolution":{"displayed":15,"state_counts":{"malformed_identifier":0,"metadata_mismatch":1,"parse_uncertain":0,"unresolved":0,"verified_exact":5,"verified_fuzzy":9},"total_outbound_references":15},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-15T06:32:42.880941+00:00","source":"crossref"},{"observed_at":"2026-08-15T06:32:39.529945+00:00","source":"retraction_watch"}],"thesis":"As of 15 August 2026, this Paper Citation Record lists 15 of 15 outbound references and 3 inbound Pith citation observations for arXiv:2605.28840."}