{"as_of":"2026-08-13T10:11:00Z","caps":{"database_statements":6,"inbound":100,"outbound":100},"context_digest":"sha256:8086b336e4170f91d84628186596d82e87fe432f055161a368046ed52fe045fb","coverage":[{"denominator":70,"lane":"reference_resolution","note":"Typed states for the displayed outbound observations.","records_observed":70,"source":"paper_references, paper_reference_links","source_observed_at":"2026-08-12T11:59:18.434546Z","state":"measured"},{"denominator":72,"lane":"standing_notices","note":"One-hop event checks from named stored sources.","records_observed":72,"source":"scholarly_work_events, retraction_status_cache","source_observed_at":"2026-08-13T06:32:02.005865+00:00","state":"measured"},{"denominator":2,"lane":"inbound_itemization","note":"Pith citing papers itemized under the disclosed page cap.","records_observed":2,"source":"paper_references, paper_reference_links","source_observed_at":"2026-06-29T10:32:47.756343Z","state":"measured"},{"denominator":1,"lane":"external_citation_measurements","note":"A source-named dated measurement, never combined with another source.","records_observed":0,"source":"arxiv_reference","source_observed_at":"2026-06-29T10:33:18.382028Z","state":"measured"}],"external_citation_measurements":[],"inbound":[{"citation":{"cited_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"cited_work":{"arxiv_id":"2411.17793","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2411.17793","snapshot_observed_at":"2026-06-29T10:33:18.382028Z","title":"Engineering ai judge systems.arXiv preprint arXiv:2411.17793,","venue":null,"work_id":"e330a923-5efb-4faf-bb9c-619db1109c48","year":2024},"citing_paper":{"arxiv_id":"2506.13538","last_updated":"2026-04-13T03:45:24Z","snapshot_observed_at":"2026-08-12T23:06:53.486968Z","submitted_at":"2025-06-16T14:26:37Z","title":"Model Context Protocol (MCP) at First Glance: Studying the Security and Maintainability of MCP Servers","version":5},"reference_index":82,"source":"pdf_text","source_observed_at":"2026-05-19T09:32:08.596259Z"},"links":{"cited_paper":"/paper/2411.17793","citing_paper":"/paper/2506.13538"},"observation_digest":"sha256:b265a1cb2c65534b8d552b2cbfa9491ca29e81f0d5563358ce56f35071dafa50","observation_id":"a1f08563-e1d9-450b-b1fb-942ab696aa28","resolution":{"observed_at":"2026-05-19T09:32:15.515399Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"cited_work":{"arxiv_id":"2411.17793","doi":null,"metadata_source":"arxiv_reference","pith_arxiv_id":"2411.17793","snapshot_observed_at":"2026-06-29T10:33:18.382028Z","title":"Engineering ai judge systems.arXiv preprint arXiv:2411.17793,","venue":null,"work_id":"e330a923-5efb-4faf-bb9c-619db1109c48","year":2024},"citing_paper":{"arxiv_id":"2606.00118","last_updated":"2026-05-27T19:42:01Z","snapshot_observed_at":"2026-08-05T19:06:05.803113Z","submitted_at":"2026-05-27T19:42:01Z","title":"An Empirical Study on Logging Evolution On Stack Overflow: Trends, Topics, and Challenges","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-06-29T10:32:47.756343Z"},"links":{"cited_paper":"/paper/2411.17793","citing_paper":"/paper/2606.00118"},"observation_digest":"sha256:86d894cfd1bd0093ede9549aacc8da9a89ae9f04900a14a0c20540e8a326a208","observation_id":"c3ec2d89-6bea-4e6b-8f3b-09f152b10195","resolution":{"observed_at":"2026-06-29T10:33:18.383835Z","resolver_source":"arxiv_id","status":"verified_exact"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}}],"links":{"evidence":"/evidence","html":"/paper/2411.17793/citation-record","integrity":"/paper/2411.17793/integrity","json":"/paper/2411.17793/citation-record.json","paper":"/paper/2411.17793"},"outbound":[{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.290245Z","title":"Claude 3 sonnet has become very lazy,","venue":null,"work_id":"4592b282-298e-42cd-b8c4-c7a9e0d5a9f3","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":1,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.139088Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:b55d853a9f684be766bfec85b15ce8e12920b5b4ee8eccae4aac8c8f197b5e1f","observation_id":"e07a633d-a329-4f9a-822d-47e8a45c6cad","resolution":{"observed_at":"2026-08-12T11:59:19.294465Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.267409Z","title":"Do you guy think the cost of gpt-4 is high,","venue":null,"work_id":"2a5dd227-ec4d-4c41-8ed9-7d4bb6f2a088","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":2,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.148769Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:1e925c9d3a6acd89d4f20cbda09c8ae08f7e5f2e43c0517ba28190a480f98db1","observation_id":"0641d58c-b4d9-45ae-a544-f469463b3b76","resolution":{"observed_at":"2026-08-12T11:59:19.271435Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.255422Z","title":"Gpt-4 is crazy expensive,","venue":null,"work_id":"bb7556f8-98f9-4737-9582-e54ea74958ea","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":3,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.152711Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:eba093683234f4266cb309d902f6a8264b3d836df52606dc435b27982069d6ee","observation_id":"7e88b40b-4104-444c-a7f7-955aa0354452","resolution":{"observed_at":"2026-08-12T11:59:19.259355Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.244068Z","title":"Open llm leaderboard,","venue":null,"work_id":"fac51eaf-7f89-4ffa-9f9b-5168786f15ce","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":4,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.157331Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:2b77b5cebde77d6788a55ddb6ce5984df4315a2c0d8a6fb9c1662c63a803a107","observation_id":"6e7e3ff0-5b01-49c5-8426-0c64801f4e06","resolution":{"observed_at":"2026-08-12T11:59:19.247901Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.231842Z","title":"Use agent metrics & llm judges to evaluate app performance,","venue":null,"work_id":"661e4c05-bde9-4656-ab37-9030b2a19b5c","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":5,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.161619Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:cd1ed1dcc1b0f53f6b7c4fd6c706f44463c7c33323bdd7f7ddf0a95899c6e70d","observation_id":"a62f18e3-1362-46cb-9a8a-d5e82ba916dd","resolution":{"observed_at":"2026-08-12T11:59:19.236198Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.219945Z","title":"2030 software engineering,","venue":null,"work_id":"5036765d-3fe1-4296-82c0-be458333edc5","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":6,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.166669Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:3e992f4b52e82389e26be2a394286fdb80b46e84e1ebbf66e401e568f5af8b90","observation_id":"2d24a91c-5b70-4273-bff3-90967b30d88b","resolution":{"observed_at":"2026-08-12T11:59:19.224133Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.206181Z","title":"The acm international conference on the foundations of software en- gineering (fse) 2024,","venue":null,"work_id":"e46c9323-2c09-408d-8feb-62d575586f79","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":7,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.170796Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:f4d467ae15ec4de9366e27b2db0e1d6546ce9d04718dfb71bf0d1b7079f848c0","observation_id":"947c5aeb-8423-4bc7-9dda-6df89c092418","resolution":{"observed_at":"2026-08-12T11:59:19.210844Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.194119Z","title":"Fm+se summit 2024,","venue":null,"work_id":"de371124-c69f-42cc-85fd-d91d74d43ced","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":8,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.175887Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:5e0e4344e01fbf7c2f7b0cfefe4ef7689c5d79586b971abe7c2f984dd87e0964","observation_id":"05ca5043-7480-4d7e-8b16-34110261baea","resolution":{"observed_at":"2026-08-12T11:59:19.198220Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.182103Z","title":"Opea initiative (open platform for enterprise ai (opea),","venue":null,"work_id":"97043aaf-82da-4eea-8696-1ca375179b61","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":9,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.180239Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:33428b641c1ceaf2f402188b9a13be2de05ce5804963965c443eed8dd9a1d14e","observation_id":"40e1b4f1-a253-4d29-a173-96e35aede6e5","resolution":{"observed_at":"2026-08-12T11:59:19.186229Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.168415Z","title":"Re-thinking data strategy and integration for artificial intelligence: concepts, opportuni- ties, and challenges,","venue":null,"work_id":"b6c193fc-483c-4579-87fa-d5f53884aad3","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":10,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.185399Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:d91d54e3ff5056674a63f63c4be2831ce5876e09d22367530960cf9f86e80c54","observation_id":"c3f25090-1dee-424a-9f4d-ef4075c739ff","resolution":{"observed_at":"2026-08-12T11:59:19.172791Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2212.08073","last_updated":"2022-12-15T06:19:23Z","snapshot_observed_at":"2026-08-02T04:53:58.766070Z","submitted_at":"2022-12-15T06:19:23Z","title":"Constitutional AI: Harmlessness from AI Feedback","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2212.08073","snapshot_observed_at":"2026-08-12T11:59:18.189110Z","title":"Constitutional ai: harmlessness from ai feedback,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":11,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.189110Z"},"links":{"cited_paper":"/paper/2212.08073","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:cdb26605931ca42f497fc91bb3d0d2f13f55e559b76b1bd9aec95856d4c3b286","observation_id":"8a957eb4-3f66-4202-b0f1-8923229c292d","resolution":{"observed_at":"2026-08-12T11:59:18.189110Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.03927","last_updated":"2024-02-22T12:32:24Z","snapshot_observed_at":"2026-08-13T04:25:14.624829Z","submitted_at":"2024-02-06T11:54:23Z","title":"Leak, Cheat, Repeat: Data Contamination and Evaluation Malpractices in Closed-Source LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.03927","snapshot_observed_at":"2026-08-12T11:59:18.194285Z","title":"Leak, cheat, repeat: data contamination and evaluation malpractices in closed-source llms,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":12,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.194285Z"},"links":{"cited_paper":"/paper/2402.03927","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:ba8b260f8ad13228516ecf45e59fa88d77bc2174a34d4e753534a1b0be090db7","observation_id":"c8023431-371b-4128-96f9-6d4dd42ca5d2","resolution":{"observed_at":"2026-08-12T11:59:18.194285Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.155055Z","title":"Meteor: an automatic metric for MT evalua- tion with improved correlation with human judgments,","venue":null,"work_id":"63f9d79d-853f-40c6-84ca-686c9b556ce3","year":2005},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":13,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.198230Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:b815f342868ece609742ca24f68bce0fcd1c5f2ff7385576be46da0ffed6c62d","observation_id":"92cf6acd-09fd-4426-b0ef-6bf32fe48823","resolution":{"observed_at":"2026-08-12T11:59:19.159428Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.143293Z","title":"Test driven development: By example addison-wesley,","venue":null,"work_id":"c3139465-1423-4643-88d2-079f7bc3b4b3","year":2002},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":14,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.201903Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:befff1ea056ca25132a74cc7ff87c3772ca25dbf1cbff3b81cc8f876dedf4654","observation_id":"cfb058a0-0668-4acf-87dd-305edac4a074","resolution":{"observed_at":"2026-08-12T11:59:19.147267Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.131649Z","title":"On the dangers of stochastic parrots: can language models be too big?","venue":null,"work_id":"ba4edb82-a945-4903-b584-bb04d08a92b7","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":15,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.205397Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:e3a5bdf97709a406d40c303af5ad928636bbd253598e4f340102e4267a962910","observation_id":"71493e67-f94e-479d-a0cd-602bac9facd1","resolution":{"observed_at":"2026-08-12T11:59:19.135774Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2303.12712","last_updated":"2023-04-13T20:41:31Z","snapshot_observed_at":"2026-08-03T04:49:15.195814Z","submitted_at":"2023-03-22T16:51:28Z","title":"Sparks of Artificial General Intelligence: Early experiments with GPT-4","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2303.12712","snapshot_observed_at":"2026-08-12T11:59:18.208969Z","title":"Sparks of artificial general intelligence: early experiments with gpt-4,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":16,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.208969Z"},"links":{"cited_paper":"/paper/2303.12712","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:f0275dd0439d6cdae1be4de78e2b97b5d01e6e3a0ddf401b851f0e5234c54af8","observation_id":"c2ac4a3a-4cff-48fd-97bf-e8fd9f8a5dd5","resolution":{"observed_at":"2026-08-12T11:59:18.208969Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2308.07201","last_updated":"2023-08-14T15:13:04Z","snapshot_observed_at":"2026-08-02T01:34:38.978920Z","submitted_at":"2023-08-14T15:13:04Z","title":"ChatEval: Towards Better LLM-based Evaluators through Multi-Agent Debate","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2308.07201","snapshot_observed_at":"2026-08-12T11:59:18.212705Z","title":"Chateval: towards better llm-based evaluators through multi-agent debate,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":17,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.212705Z"},"links":{"cited_paper":"/paper/2308.07201","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:0fe86916fe2418e3fb0f5a4ae44b697ce58ce22078a3056c5a68ad77095e8eb8","observation_id":"62e3339c-7943-452c-847f-8cbde46d5111","resolution":{"observed_at":"2026-08-12T11:59:18.212705Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.118603Z","title":"A survey on evaluation of large language models,","venue":null,"work_id":"103cb2e5-27a5-4dc5-a59b-729158cb0f94","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":18,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.217544Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:acfb13da0f07697c1e93a92912b1101cd95f9c9663d2c7821d8f4f4db48b0d9e","observation_id":"0935c594-8e41-416a-95db-9f86a061e380","resolution":{"observed_at":"2026-08-12T11:59:19.122927Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.14735","last_updated":"2025-05-11T09:23:41Z","snapshot_observed_at":"2026-08-13T05:43:18.634746Z","submitted_at":"2023-10-23T09:15:18Z","title":"Unleashing the potential of prompt engineering for large language models","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.14735","snapshot_observed_at":"2026-08-12T11:59:18.221024Z","title":"Unleashing the potential of prompt engineering in large language models: a comprehensive review,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":19,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.221024Z"},"links":{"cited_paper":"/paper/2310.14735","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:e05fbff358e67576ca0841359f0bbf580f1b8959b4041b304f87ecb9c89cbacd","observation_id":"e65b98a8-a1b8-4100-a638-188fd934f478","resolution":{"observed_at":"2026-08-12T11:59:18.221024Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.106443Z","title":"Towards training reproducible deep learning models,","venue":null,"work_id":"f9325a4b-74f4-4c80-b6a1-690d1f14c850","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":20,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.224849Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:18045853ade453eec9e2d36dfab38ef764a20f32d0d4765ae6696094018b720a","observation_id":"a970bc9c-49d3-465b-9583-2f2c3cd9b86e","resolution":{"observed_at":"2026-08-12T11:59:19.110698Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2402.10669","last_updated":"2024-09-26T03:16:52Z","snapshot_observed_at":"2026-08-13T04:17:35.878300Z","submitted_at":"2024-02-16T13:21:06Z","title":"Humans or LLMs as the Judge? A Study on Judgement Biases","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.10669","snapshot_observed_at":"2026-08-12T11:59:18.228507Z","title":"Humans or llms as the judge? a study on judgement biases,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":21,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.228507Z"},"links":{"cited_paper":"/paper/2402.10669","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:284349c230848634f6c9c0898a6bd7637c2ef2ae99d1e2279f3851d086df1e39","observation_id":"d2fa95c7-8c31-498e-8bdf-dec0558784c8","resolution":{"observed_at":"2026-08-12T11:59:18.228507Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2307.09009","last_updated":"2023-10-31T16:13:44Z","snapshot_observed_at":"2026-08-10T06:43:50.786180Z","submitted_at":"2023-07-18T06:56:08Z","title":"How is ChatGPT's behavior changing over time?","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2307.09009","snapshot_observed_at":"2026-08-12T11:59:18.233491Z","title":"How is chatgpt’s behavior changing over time?","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":22,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.233491Z"},"links":{"cited_paper":"/paper/2307.09009","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:363a5586d47deba9dab3c2256696005484ae9e6a295a251ff7accdc3cb12ed0d","observation_id":"c5ad56e2-3a5c-4c91-a84c-08c4ee58a9c2","resolution":{"observed_at":"2026-08-12T11:59:18.233491Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2403.04132","last_updated":"2024-03-07T01:22:38Z","snapshot_observed_at":"2026-08-02T17:55:33.750637Z","submitted_at":"2024-03-07T01:22:38Z","title":"Chatbot Arena: An Open Platform for Evaluating LLMs by Human Preference","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2403.04132","snapshot_observed_at":"2026-08-12T11:59:18.237211Z","title":"Chatbot arena: an open platform for evaluating llms by human preference,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":23,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.237211Z"},"links":{"cited_paper":"/paper/2403.04132","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:b8cead4de8bf2d093a7d9933f5f421a78841198b534dd2dd2f2177a7531ba28a","observation_id":"64e25685-2f9e-4c1b-9113-cc8e95572a23","resolution":{"observed_at":"2026-08-12T11:59:18.237211Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.278962Z","title":"Available: https://www.reddit.com/r/ClaudeAI/comments/ 1bv8ww5/claude 3 sonnet has become very lazy/","venue":null,"work_id":"7e85d1df-75fd-44df-97b5-ba311af5e62b","year":null},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":24,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.143956Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:2724494e3f600f8f33b5527ff2a1be0c91f77dd50a68ac3dfdb3fc5eaade53a6","observation_id":"2ddc37fd-3306-4aac-bcb2-a6cc389bc29f","resolution":{"observed_at":"2026-08-12T11:59:19.282971Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2110.14168","last_updated":"2021-11-18T00:23:45Z","snapshot_observed_at":"2026-08-07T01:45:38.840969Z","submitted_at":"2021-10-27T04:49:45Z","title":"Training Verifiers to Solve Math Word Problems","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2110.14168","snapshot_observed_at":"2026-08-12T11:59:18.241194Z","title":"Training verifiers to solve math word problems,","venue":null,"work_id":null,"year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":25,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.241194Z"},"links":{"cited_paper":"/paper/2110.14168","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:113104859f6cb3980259f6105079093384b27e7309807873614bb158a46401bd","observation_id":"10d1d72c-38ce-4f89-8809-379d44341fc4","resolution":{"observed_at":"2026-08-12T11:59:18.241194Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.093882Z","title":"Evalullm: llm assisted evaluation of generative outputs,","venue":null,"work_id":"d98933fb-0bcd-49f3-b580-fe42518b476d","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":26,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.245859Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:d554242e60d0674f08298353c1fd7b4a7b97611282673b11594875b80194371e","observation_id":"54a73f2e-70a0-43f2-89e7-684da527c5e6","resolution":{"observed_at":"2026-08-12T11:59:19.098115Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.081950Z","title":"Qlora: ef- ficient finetuning of quantized llms,","venue":null,"work_id":"769cd384-bcab-4e72-ac32-bf6452814be5","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":27,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.249801Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:f1a0bb929e98193351c71bd05d182f4f9c451ac6ff823074f2d396d4237565e8","observation_id":"2ab6465c-c725-4a04-b939-394da8855761","resolution":{"observed_at":"2026-08-12T11:59:19.086396Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.071374Z","title":"Fira: fine-grained graph-based code change representation for automated com- mit message generation,","venue":null,"work_id":"9692b42e-47c9-4a56-983b-256081c93f66","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":28,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.253234Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:9d20ada1fc25168a720fb243794b464d3153644d07caa16372e79ea27c0cfd10","observation_id":"aec2f2f2-46ed-4a8e-9983-d24f1b299583","resolution":{"observed_at":"2026-08-12T11:59:19.075156Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.059979Z","title":"Alpacafarm: a simulation framework for methods that learn from human feedback,","venue":null,"work_id":"8b582c22-ffbb-40c8-bacd-ed8b20487f81","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":29,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.256879Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:9c253547f9cd638af5dff7d6a5144df8489b5f212fb40dd1bb47222c7ec0010e","observation_id":"2c4f6a65-7ce3-4ec6-a895-0cbd0da38f37","resolution":{"observed_at":"2026-08-12T11:59:19.063807Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.048693Z","title":"Bias and fairness in large language models: a survey,","venue":null,"work_id":"64ea2262-b710-4b08-87cc-a16d07fe501b","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":30,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.260862Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:93c5ce36354bdc22e7cf6aca5df930062b91347cc5847399a95a651c7ef97d9a","observation_id":"9831402a-94fc-4b4a-bb73-8d52cc27b3c7","resolution":{"observed_at":"2026-08-12T11:59:19.052740Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2205.12679","last_updated":"2023-02-26T08:14:05Z","snapshot_observed_at":"2026-08-13T04:53:18.209120Z","submitted_at":"2022-05-25T11:38:48Z","title":"Self-Guided Noise-Free Data Generation for Efficient Zero-Shot Learning","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2205.12679","snapshot_observed_at":"2026-08-12T11:59:18.266978Z","title":"Self-guided noise- free data generation for efficient zero-shot learning,","venue":null,"work_id":null,"year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":31,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.266978Z"},"links":{"cited_paper":"/paper/2205.12679","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:313fb4cabb947661b4e4ade14c81f577c87dab2a7028507675b00d04a09c19e0","observation_id":"535c660a-2eb5-4e4a-8c8c-2d803839f901","resolution":{"observed_at":"2026-08-12T11:59:18.266978Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.01383","last_updated":"2025-05-14T06:05:53Z","snapshot_observed_at":"2026-08-13T04:28:25.209847Z","submitted_at":"2024-02-02T13:06:35Z","title":"LLM-based NLG Evaluation: Current Status and Challenges","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.01383","snapshot_observed_at":"2026-08-12T11:59:18.271869Z","title":"Llm-based nlg evaluation: current status and challenges,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":32,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.271869Z"},"links":{"cited_paper":"/paper/2402.01383","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:3f46fa49af8d439e5f03f7147eac7f2961c1865266ff8dcb85c97adc43a0680f","observation_id":"4944e0f9-35d0-4078-8829-e8f5a610a8e9","resolution":{"observed_at":"2026-08-12T11:59:18.271869Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.037040Z","title":"Fm+se vision 2030,","venue":null,"work_id":"53583729-09e3-4843-9016-ac6c98f39393","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":33,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.276954Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:efeae8b2d69fe304e3dc394c3a9404f62df72e6c96f45d6d1ee36acf3e0a2ee4","observation_id":"964d0df2-5403-4e1b-8136-a64903194b90","resolution":{"observed_at":"2026-08-12T11:59:19.041370Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.025221Z","title":"Rethinking software engineering in the foundation model era: a curated catalogue of challenges in the development of trustworthy fmware,","venue":null,"work_id":"d1f1481e-bfe5-43bf-bceb-654eafca54b7","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":34,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.280771Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:f5258577d722153092d4183df72918f62a02eb7fdc61cffa64b404b19400b520","observation_id":"80f2e391-f7a0-4f0b-9247-39c261d7d047","resolution":{"observed_at":"2026-08-12T11:59:19.029239Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2009.03300","last_updated":"2021-01-12T18:57:11Z","snapshot_observed_at":"2026-08-10T12:35:09.020030Z","submitted_at":"2020-09-07T17:59:25Z","title":"Measuring Massive Multitask Language Understanding","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2009.03300","snapshot_observed_at":"2026-08-12T11:59:18.285054Z","title":"Measuring massive multitask language understanding,","venue":null,"work_id":null,"year":2009},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":35,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.285054Z"},"links":{"cited_paper":"/paper/2009.03300","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:8670c74a8c0a850f86ea172731ed3978ea28eebda48bd0c25e372ffce5a7de65","observation_id":"38c394b4-fb5d-4586-b09d-45b121f5a5ef","resolution":{"observed_at":"2026-08-12T11:59:18.285054Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2311.05232","last_updated":"2024-11-19T12:42:45Z","snapshot_observed_at":"2026-07-06T16:45:07.733095Z","submitted_at":"2023-11-09T09:25:37Z","title":"A Survey on Hallucination in Large Language Models: Principles, Taxonomy, Challenges, and Open Questions","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2311.05232","snapshot_observed_at":"2026-08-12T11:59:18.289124Z","title":"A survey on hallucination in large language models: principles, taxonomy, challenges, and open questions,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":36,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.289124Z"},"links":{"cited_paper":"/paper/2311.05232","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:2a48adeb10926c5f772a94a2f3f43c2af399535b99dcf848de64a3f0fc68219b","observation_id":"abc6d780-04f7-4f85-bcb2-8b0d8f8b217b","resolution":{"observed_at":"2026-08-12T11:59:18.289124Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"1805.00899","last_updated":"2018-10-22T17:36:07Z","snapshot_observed_at":"2026-08-02T15:33:17.783178Z","submitted_at":"2018-05-02T16:27:32Z","title":"AI safety via debate","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"1805.00899","snapshot_observed_at":"2026-08-12T11:59:18.293117Z","title":"Ai safety via debate,","venue":null,"work_id":null,"year":2018},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":37,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.293117Z"},"links":{"cited_paper":"/paper/1805.00899","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:018a37630c393db108971f2a7ac44ad4cae1690b7ab401cc5caadc6cfdf0e861","observation_id":"8f6c5c9f-b355-4868-9765-adf08a204dd1","resolution":{"observed_at":"2026-08-12T11:59:18.293117Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.014434Z","title":"Kejriwal, Domain-specific knowledge graph construction","venue":null,"work_id":"932d745e-16b8-424a-9a0e-71c135db553c","year":2019},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":38,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.297827Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:11abc49bb622a049ece3a8fb77cca03af9536e2d310bb3f46ec6b0f22a292392","observation_id":"601eb689-23ad-4590-b3ee-368c505574dd","resolution":{"observed_at":"2026-08-12T11:59:19.018229Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.04622","last_updated":"2024-07-12T16:38:12Z","snapshot_observed_at":"2026-08-12T23:28:03.534233Z","submitted_at":"2024-07-05T16:29:15Z","title":"On scalable oversight with weak LLMs judging strong LLMs","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.04622","snapshot_observed_at":"2026-08-12T11:59:18.301527Z","title":"On scalable oversight with weak llms judging strong llms,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":39,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.301527Z"},"links":{"cited_paper":"/paper/2407.04622","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:189cc4fa9d8658295b9cffa840907588b5728e25ffbb04fd61d59ad0cf3886e6","observation_id":"dce9d1bd-1dd9-479a-9f89-db458e0f2469","resolution":{"observed_at":"2026-08-12T11:59:18.301527Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2402.06782","last_updated":"2024-07-25T23:32:21Z","snapshot_observed_at":"2026-08-13T04:22:15.640251Z","submitted_at":"2024-02-09T21:05:01Z","title":"Debating with More Persuasive LLMs Leads to More Truthful Answers","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2402.06782","snapshot_observed_at":"2026-08-12T11:59:18.305344Z","title":"Debating with more persuasive llms leads to more truthful answers,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":40,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.305344Z"},"links":{"cited_paper":"/paper/2402.06782","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:39b169e1c1f33f732322b3cff9ee28721eaebf72a2dbdfbb3055bc28837b5107","observation_id":"28284a8d-410a-4e89-b44b-1d1c75aff2f3","resolution":{"observed_at":"2026-08-12T11:59:18.305344Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:19.002278Z","title":"Dspy: compiling declarative language model calls into state-of-the-art pipelines,","venue":null,"work_id":"0f5a83d8-48a2-4036-9f44-c907cab86e01","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":41,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.309227Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:d2e609324afde65466e4a5b7ba6efbfc1c4f8bb65fcbb60fefbcf82042906e37","observation_id":"84a337a4-3562-4833-9efb-ddcd9b7d00f5","resolution":{"observed_at":"2026-08-12T11:59:19.007287Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.989500Z","title":"Software engineering for machine learning applications (semla) 2023,","venue":null,"work_id":"b8c86638-1174-401e-910b-d771b71f7f0e","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":42,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.313972Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:6b99276caf72873409a091f8c40a8efa36ff7367027093c405163f9ce717c184","observation_id":"f9386766-e1fd-461f-9013-e120f2ed1260","resolution":{"observed_at":"2026-08-12T11:59:18.993839Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2310.06452","last_updated":"2024-02-19T14:39:07Z","snapshot_observed_at":"2026-07-29T18:39:02.141391Z","submitted_at":"2023-10-10T09:25:44Z","title":"Understanding the Effects of RLHF on LLM Generalisation and Diversity","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.06452","snapshot_observed_at":"2026-08-12T11:59:18.317823Z","title":"Understanding the effects of rlhf on llm generalisation and diversity,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":43,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.317823Z"},"links":{"cited_paper":"/paper/2310.06452","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:87fe099e754f027951fdf4fdbab73ba0013e168c61b36bd1912c82ad214b5d28","observation_id":"dac9e283-2527-47fb-8842-c2df3acbb6cc","resolution":{"observed_at":"2026-08-12T11:59:18.317823Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.976977Z","title":"On the role of knowledge graphs in explainable ai,","venue":null,"work_id":"fe06dc94-cb34-4077-a852-b1b994d615d5","year":2020},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":44,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.322382Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:4945f95477df2e5551831bbc2bd10fbd3375676b0c3ced7ad71237831052656a","observation_id":"3aec817e-5fca-4029-af5c-2709ab851d84","resolution":{"observed_at":"2026-08-12T11:59:18.981263Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2305.19118","last_updated":"2024-10-09T02:41:21Z","snapshot_observed_at":"2026-08-01T16:20:31.337598Z","submitted_at":"2023-05-30T15:25:45Z","title":"Encouraging Divergent Thinking in Large Language Models through Multi-Agent Debate","version":4},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2305.19118","snapshot_observed_at":"2026-08-12T11:59:18.326017Z","title":"Encouraging divergent thinking in large language models through multi-agent debate,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":45,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.326017Z"},"links":{"cited_paper":"/paper/2305.19118","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:3911a2dafda0f097329cc9e7489ea88b365707448b26653f5aca58b12839e9ac","observation_id":"2104f4a8-8d00-479d-aa47-f19387188b8c","resolution":{"observed_at":"2026-08-12T11:59:18.326017Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.964495Z","title":"Rouge: a package for automatic evaluation of summaries,","venue":null,"work_id":"c05414de-b77e-4eca-a4d3-7556cb6a77e5","year":2004},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":46,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.330973Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:bd5939cce08407eca9845f393ba4891b4dd3eb81b27267c0242b5ecd44dfc197","observation_id":"9724063b-752b-41cc-92a5-2e8b9e3fa414","resolution":{"observed_at":"2026-08-12T11:59:18.968535Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.07503","last_updated":"2024-08-10T20:46:47Z","snapshot_observed_at":"2026-08-13T00:32:55.965962Z","submitted_at":"2024-04-11T06:34:17Z","title":"Best Practices and Lessons Learned on Synthetic Data","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.07503","snapshot_observed_at":"2026-08-12T11:59:18.334631Z","title":"Best practices and lessons learned on synthetic data for language models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":47,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.334631Z"},"links":{"cited_paper":"/paper/2404.07503","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:ad2f4e0488dc2755a66cc2b0663d6edbe65e3ceb1cf0e1d6885c187949017ec8","observation_id":"56a5924d-8edc-4c5f-8af5-dc72367b80be","resolution":{"observed_at":"2026-08-12T11:59:18.334631Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2309.13308","last_updated":"2023-09-23T08:46:11Z","snapshot_observed_at":"2026-08-13T10:07:09.184755Z","submitted_at":"2023-09-23T08:46:11Z","title":"Calibrating LLM-Based Evaluator","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2309.13308","snapshot_observed_at":"2026-08-12T11:59:18.339313Z","title":"Calibrating llm-based evaluator,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":48,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.339313Z"},"links":{"cited_paper":"/paper/2309.13308","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:1c22fe62da4f504f18419ac1de2f8df4d52f7c984112bb7776c8b2538d404180","observation_id":"a57f28fd-7676-4364-9c5f-b975c931315e","resolution":{"observed_at":"2026-08-12T11:59:18.339313Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.952500Z","title":"Generating training data with language models: towards zero-shot language understanding,","venue":null,"work_id":"79b14e01-1eae-4969-8ad7-d92f35a32f57","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":49,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.343429Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:5e2221383400fcb0945e7c8600dc8137bcc5d27303fd6a455a1ede24ec21d109","observation_id":"ae9b3172-cb8b-4264-b00e-b1fc5795ef68","resolution":{"observed_at":"2026-08-12T11:59:18.956636Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.940463Z","title":"Cider: robust consensus-based image description evaluation,","venue":null,"work_id":"c64e4fe6-6856-48f5-ac51-0d2d3847df0c","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":50,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.348047Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:9d67a97231e7a84f56adfe53a8715c324f7b89182d9bc1df3222e67503fda02e","observation_id":"eb717707-4edb-4c03-aeb5-23bf499ef0a4","resolution":{"observed_at":"2026-08-12T11:59:18.944650Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.927793Z","title":"A framework for evaluating and improving requirements specifications based on the developers and testers perspective,","venue":null,"work_id":"5193e3b9-38a2-40d6-bc3f-db546cdc30da","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":51,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.352122Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:95e707e2ce9be6320f6e6e1601a8829dfed97bd43dc7eccadfdc5294b79b2ede","observation_id":"95a55474-bc6c-4ec1-81f6-73192c62e639","resolution":{"observed_at":"2026-08-12T11:59:18.932435Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2407.03479","last_updated":"2024-07-03T19:53:47Z","snapshot_observed_at":"2026-08-12T23:29:17.040194Z","submitted_at":"2024-07-03T19:53:47Z","title":"Human-Centered Design Recommendations for LLM-as-a-Judge","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.03479","snapshot_observed_at":"2026-08-12T11:59:18.356623Z","title":"Human-centered design recommendations for llm-as-a-judge,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":52,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.356623Z"},"links":{"cited_paper":"/paper/2407.03479","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:7023a273f454c0a382320d77f7557a84b169e2afafbd9682542f3445d0560dd7","observation_id":"698c9c48-3d7c-4663-9190-028b2013d533","resolution":{"observed_at":"2026-08-12T11:59:18.356623Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.915368Z","title":"Bleu: a method for automatic evaluation of machine translation,","venue":null,"work_id":"83134da3-ae6d-4f7c-a78e-faae5eb88a75","year":2002},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":53,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.360737Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:ed3200981a539b558f3b14e92dd23d72ddb0deb00d34fb35867b83ce1052542d","observation_id":"d380be56-886c-43af-a80c-d04386398aaf","resolution":{"observed_at":"2026-08-12T11:59:18.920316Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2408.13296","last_updated":"2024-10-30T01:04:15Z","snapshot_observed_at":"2026-08-12T22:58:36.142991Z","submitted_at":"2024-08-23T14:48:02Z","title":"The Ultimate Guide to Fine-Tuning LLMs from Basics to Breakthroughs: An Exhaustive Review of Technologies, Research, Best Practices, Applied Research Challenges and Opportunities","version":3},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13296","snapshot_observed_at":"2026-08-12T11:59:18.365781Z","title":null,"venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":54,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.365781Z"},"links":{"cited_paper":"/paper/2408.13296","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:ba8e2847488dcc00a9bef6e2527b7629b0c5fde557c46c3b32ad2dd08407f329","observation_id":"a5a90d46-9e76-4ecb-94ce-4bf898a63156","resolution":{"observed_at":"2026-08-12T11:59:18.365781Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2310.10076","last_updated":"2023-10-16T05:19:02Z","snapshot_observed_at":"2026-08-13T05:48:49.313792Z","submitted_at":"2023-10-16T05:19:02Z","title":"Verbosity Bias in Preference Labeling by Large Language Models","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2310.10076","snapshot_observed_at":"2026-08-12T11:59:18.369902Z","title":"Verbosity bias in preference labeling by large language models,","venue":null,"work_id":null,"year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":55,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.369902Z"},"links":{"cited_paper":"/paper/2310.10076","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:162192bd4538e480c13d87217329dd680d50a59e386c472292fbc3cbf7ed6421","observation_id":"57920572-3461-49cf-b1b9-5a97faa05cee","resolution":{"observed_at":"2026-08-12T11:59:18.369902Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2004.04696","last_updated":"2020-05-21T16:53:47Z","snapshot_observed_at":"2026-08-10T19:14:01.245716Z","submitted_at":"2020-04-09T17:26:52Z","title":"BLEURT: Learning Robust Metrics for Text Generation","version":5},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2004.04696","snapshot_observed_at":"2026-08-12T11:59:18.374841Z","title":"Bleurt: learning robust metrics for text generation,","venue":null,"work_id":null,"year":2004},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":56,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.374841Z"},"links":{"cited_paper":"/paper/2004.04696","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:2d5a3ce7ae895e08704c105840124a4a6d14c6d2f1ebe18b271be6c64e879e21","observation_id":"0f1b10a0-d706-48ae-a6fd-e0e483db6f0e","resolution":{"observed_at":"2026-08-12T11:59:18.374841Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.902491Z","title":"On automatic summarization of what and why information in source code changes,","venue":null,"work_id":"cd6cdfce-ad2c-4c13-bf26-310ae90dd5c5","year":2016},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":57,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.379788Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:464990f320028aef20a89e8fa8bc665ecbfdc6cacc7b6d2f2ee9e61c970b551e","observation_id":"1cd8717b-f6b1-4089-aec4-d5b796dbcae0","resolution":{"observed_at":"2026-08-12T11:59:18.906996Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.890035Z","title":"On the evaluation of neural code summarization,","venue":null,"work_id":"047771d6-7d01-48c2-9e3b-a9ca60b338da","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":58,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.383721Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:245ad4c5921c015186787040ea6ff46425f414e165c4364bd12ef0c2c18c6d3f","observation_id":"ff1d24d0-ab0f-4ea3-92d4-57d3d1841f88","resolution":{"observed_at":"2026-08-12T11:59:18.894287Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.878433Z","title":"RACE: Retrieval-augmented commit message generation,","venue":null,"work_id":"58e2f551-2991-4b33-a25a-a0a7f35e1954","year":2022},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":59,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.388673Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:0976eb3d8c31d2c20f1856ca3b45a2f6cd89842a88995fef0b683a49ab6cb35f","observation_id":"f1f2be3f-b8ef-4e6c-9a72-9c183b1bb0f9","resolution":{"observed_at":"2026-08-12T11:59:18.882388Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.866510Z","title":"On the evaluation of commit message generation models: an experimental study,","venue":null,"work_id":"b9891dce-5e2e-4133-84ea-347ce1c49822","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":60,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.392463Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:788dbb127b3b1df300250e14cb584b59070cf9b03d7910f820b16c5649faf8b6","observation_id":"9de57533-72ab-447b-9c2e-9a16d1936911","resolution":{"observed_at":"2026-08-12T11:59:18.870426Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.852877Z","title":"Alpaca: a strong, replicable instruction-following model,","venue":null,"work_id":"d04c6ba1-2f21-4a22-aecf-e9fdbde89075","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":61,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.397190Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:8e080b1a7bb880c6c1f479980664cd4e3338bef840e9385baa4d979f07c9ed22","observation_id":"68bbccb3-869f-49bb-aa65-0f68b69ef797","resolution":{"observed_at":"2026-08-12T11:59:18.858724Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2406.12624","last_updated":"2025-08-18T00:07:23Z","snapshot_observed_at":"2026-08-12T23:40:03.945084Z","submitted_at":"2024-06-18T13:49:54Z","title":"Judging the Judges: Evaluating Alignment and Vulnerabilities in LLMs-as-Judges","version":6},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2406.12624","snapshot_observed_at":"2026-08-12T11:59:18.400788Z","title":"Judging the judges: evaluating alignment and vulnerabilities in llms-as-judges,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":62,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.400788Z"},"links":{"cited_paper":"/paper/2406.12624","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:77c784edd9a02778366ed9204252fd7d1fff071c418635a8c6c1dc56fb994bf8","observation_id":"196eee81-2147-4992-9d96-4f60defcddda","resolution":{"observed_at":"2026-08-12T11:59:18.400788Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.840041Z","title":"Synthetic data, real errors: how (not) to publish and use synthetic data,","venue":null,"work_id":"168b336f-8c65-43a5-9de0-f19821c4f64a","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":63,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.405510Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:5db9a39d7a512ef5113ab13a20245f8bfdb2061a7808ba6d95c001327c23028e","observation_id":"d64aa7a2-c591-4f8b-8884-5cb6100a9cc3","resolution":{"observed_at":"2026-08-12T11:59:18.844266Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":{"arxiv_id":"2404.18796","last_updated":"2024-05-01T15:37:11Z","snapshot_observed_at":"2026-07-06T02:11:23.670680Z","submitted_at":"2024-04-29T15:33:23Z","title":"Replacing Judges with Juries: Evaluating LLM Generations with a Panel of Diverse Models","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2404.18796","snapshot_observed_at":"2026-08-12T11:59:18.409126Z","title":"Replacing judges with juries: evaluating llm generations with a panel of diverse models,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":64,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.409126Z"},"links":{"cited_paper":"/paper/2404.18796","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:2733f70684fac470bb9b6f77b8d0a4fc03fcea33d30370c33a2d84f94bcf3b2e","observation_id":"a7d2c3be-6852-4eea-b3be-9e62162e1d76","resolution":{"observed_at":"2026-08-12T11:59:18.409126Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2408.13006","last_updated":"2025-03-30T17:59:47Z","snapshot_observed_at":"2026-08-12T22:58:43.065552Z","submitted_at":"2024-08-23T11:49:01Z","title":"Systematic Evaluation of LLM-as-a-Judge in LLM Alignment Tasks: Explainable Metrics and Diverse Prompt Templates","version":2},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2408.13006","snapshot_observed_at":"2026-08-12T11:59:18.413886Z","title":"Systematic evaluation of llm-as-a-judge in llm alignment tasks: explainable metrics and diverse prompt templates,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":65,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.413886Z"},"links":{"cited_paper":"/paper/2408.13006","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:daa905ab4a2b21b8e0dcadea2b411f331c8fe082216fb8fb4b896d3139a772ce","observation_id":"02d0b65c-f5f1-4087-b51f-22bad0f786a2","resolution":{"observed_at":"2026-08-12T11:59:18.413886Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":{"arxiv_id":"2407.04014","last_updated":"2024-07-04T15:45:15Z","snapshot_observed_at":"2026-08-12T23:28:42.103876Z","submitted_at":"2024-07-04T15:45:15Z","title":"Offline Energy-Optimal LLM Serving: Workload-Based Energy Models for LLM Inference on Heterogeneous Systems","version":1},"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":null,"pith_arxiv_id":"2407.04014","snapshot_observed_at":"2026-08-12T11:59:18.417892Z","title":"Offline energy-optimal llm serv- ing: Workload-based energy models for llm inference on heterogeneous systems,","venue":null,"work_id":null,"year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":66,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.417892Z"},"links":{"cited_paper":"/paper/2407.04014","citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:e31abc4debc9a210ba31d1d680b6721182f13303a7e1bebcecbf5f3ae22934df","observation_id":"5746f32f-31dc-40ec-b425-a3aa15be497f","resolution":{"observed_at":"2026-08-12T11:59:18.417892Z","resolver_source":null,"status":"unresolved"},"standing_notice":{"events":[],"reason":"canonical_work_link_unavailable","source_receipts":[],"state":"unavailable"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.827971Z","title":"Fake it till you make it: face analysis in the wild using synthetic data alone,","venue":null,"work_id":"14449594-5829-489c-ab70-c3cc68e7297b","year":2021},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":67,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.422862Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:5ebbb2d16e7410be6ef588b513fa6e720b163a89b8533f72b8d5740582fc835c","observation_id":"99eedc02-be11-4d1c-bb22-864eaef6e7e4","resolution":{"observed_at":"2026-08-12T11:59:18.831987Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.816236Z","title":"Commit message generation via chatgpt: how far are we?","venue":null,"work_id":"31fa2c1e-6af4-4804-97d1-da4127d5e13d","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":68,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.427237Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:30b2b7c53e2d3b699f9be0e97724fa9b4f3562f11b15082fe25e5179ef6516fe","observation_id":"219aebda-04d4-455d-b19a-6145c4cbea0a","resolution":{"observed_at":"2026-08-12T11:59:18.820449Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.804402Z","title":"Automatic commit message generation: a critical review and directions for future work,","venue":null,"work_id":"c509f08b-9099-4fdc-97b6-67c5499530a6","year":2024},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":69,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.431034Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:1ad25a39cd0602cf68ab4fcef8bca36c5d63dc6b6d85bd133fe476a23db42eee","observation_id":"f890a40e-3532-4f3d-b0f0-0e99dbe9cb2c","resolution":{"observed_at":"2026-08-12T11:59:18.808693Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}},{"citation":{"cited_paper":null,"cited_work":{"arxiv_id":null,"doi":null,"metadata_source":"raw_reference","pith_arxiv_id":null,"snapshot_observed_at":"2026-08-12T11:59:18.789857Z","title":"Judging llm-as-a-judge with mt-bench and chatbot arena,","venue":null,"work_id":"4a985510-5646-4281-870e-1538dfbe5f9e","year":2023},"citing_paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems","version":1},"reference_index":70,"source":"pdf_text","source_observed_at":"2026-08-12T11:59:18.434546Z"},"links":{"citing_paper":"/paper/2411.17793"},"observation_digest":"sha256:a8eefbd6442be58482789a96d9312f1037ec814950363cd2e691543e4fb16889","observation_id":"b3e532f0-e1c9-45c0-8e81-6851a8f47262","resolution":{"observed_at":"2026-08-12T11:59:18.795757Z","resolver_source":"raw_fallback","status":"verified_fuzzy"},"standing_notice":{"events":[],"observation":"No event found in the named queried sources as of 2026-08-13T06:32:02.005865+00:00.","reason":null,"source_receipts":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"state":"measured"}}],"paper":{"arxiv_id":"2411.17793","last_updated":"2024-11-26T17:43:32Z","latest_version":1,"primary_category":"cs.SE","snapshot_observed_at":"2026-08-12T20:18:05.272256Z","submitted_at":"2024-11-26T17:43:32Z","title":"Engineering AI Judge Systems"},"reference_resolution":{"displayed":70,"state_counts":{"malformed_identifier":0,"metadata_mismatch":0,"parse_uncertain":0,"unresolved":28,"verified_exact":0,"verified_fuzzy":42},"total_outbound_references":70},"refusal":"A citation records a reference. It does not transfer a finding from one paper to another.","schema":"pith.paper-citation-record.v1","standing_sources":[{"observed_at":"2026-08-13T06:32:02.005865+00:00","source":"crossref"},{"observed_at":"2026-08-13T06:31:53.387327+00:00","source":"retraction_watch"}],"thesis":"As of 13 August 2026, this Paper Citation Record lists 70 of 70 outbound references and 2 inbound Pith citation observations for arXiv:2411.17793."}